(self)
| 134 | ) |
| 135 | |
| 136 | def _check_warning(self) -> None: |
| 137 | if self.warned: # warn only once |
| 138 | return |
| 139 | assert self.crawler.stats |
| 140 | if self.get_virtual_size() > self.warning: |
| 141 | self.crawler.stats.set_value("memusage/warning_reached", 1) |
| 142 | self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) |
| 143 | mem = self.warning / 1024 / 1024 |
| 144 | logger.warning( |
| 145 | "Memory usage reached %(memusage)dMiB", |
| 146 | {"memusage": mem}, |
| 147 | extra={"crawler": self.crawler}, |
| 148 | ) |
| 149 | if self.notify_mails: |
| 150 | subj = ( |
| 151 | f"{self.crawler.settings['BOT_NAME']} warning: " |
| 152 | f"memory usage reached {mem}MiB at {socket.gethostname()}" |
| 153 | ) |
| 154 | self._send_report(self.notify_mails, subj) |
| 155 | self.crawler.stats.set_value("memusage/warning_notified", 1) |
| 156 | self.warned = True |
| 157 | |
| 158 | def _send_report(self, rcpts: list[str], subject: str) -> None: |
| 159 | """send notification mail with some additional useful info""" |
nothing calls this directly
no test coverage detected