| 22 | |
| 23 | |
| 24 | class StatsCollector: |
| 25 | def __init__(self, crawler: Crawler): |
| 26 | self._dump: bool = crawler.settings.getbool("STATS_DUMP") |
| 27 | self._stats: StatsT = {} |
| 28 | self._crawler: Crawler = crawler |
| 29 | |
| 30 | def __getattribute__(self, name: str) -> Any: |
| 31 | cached_name = f"_cached_{name}" |
| 32 | try: |
| 33 | return super().__getattribute__(cached_name) |
| 34 | except AttributeError: |
| 35 | pass |
| 36 | |
| 37 | original_attr = super().__getattribute__(name) |
| 38 | |
| 39 | if name in { |
| 40 | "get_value", |
| 41 | "get_stats", |
| 42 | "set_value", |
| 43 | "set_stats", |
| 44 | "inc_value", |
| 45 | "max_value", |
| 46 | "min_value", |
| 47 | "clear_stats", |
| 48 | "open_spider", |
| 49 | "close_spider", |
| 50 | } and callable(original_attr): |
| 51 | wrapped = _warn_spider_arg(original_attr) |
| 52 | setattr(self, cached_name, wrapped) |
| 53 | return wrapped |
| 54 | |
| 55 | return original_attr |
| 56 | |
| 57 | def get_value( |
| 58 | self, key: str, default: Any = None, spider: Spider | None = None |
| 59 | ) -> Any: |
| 60 | return self._stats.get(key, default) |
| 61 | |
| 62 | def get_stats(self, spider: Spider | None = None) -> StatsT: |
| 63 | return self._stats |
| 64 | |
| 65 | def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: |
| 66 | self._stats[key] = value |
| 67 | |
| 68 | def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: |
| 69 | self._stats = stats |
| 70 | |
| 71 | def inc_value( |
| 72 | self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None |
| 73 | ) -> None: |
| 74 | d = self._stats |
| 75 | d[key] = d.setdefault(key, start) + count |
| 76 | |
| 77 | def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: |
| 78 | self._stats[key] = max(self._stats.setdefault(key, value), value) |
| 79 | |
| 80 | def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: |
| 81 | self._stats[key] = min(self._stats.setdefault(key, value), value) |
no outgoing calls