\ Reap workers to avoid zombie processes
(self)
| 595 | self.kill_worker(pid, signal.SIGKILL) |
| 596 | |
| 597 | def reap_workers(self): |
| 598 | """\ |
| 599 | Reap workers to avoid zombie processes |
| 600 | """ |
| 601 | try: |
| 602 | while True: |
| 603 | wpid, status = os.waitpid(-1, os.WNOHANG) |
| 604 | if not wpid: |
| 605 | break |
| 606 | if self.reexec_pid == wpid: |
| 607 | self.reexec_pid = 0 |
| 608 | else: |
| 609 | # A worker was terminated. If the termination reason was |
| 610 | # that it could not boot, we'll shut it down to avoid |
| 611 | # infinite start/stop cycles. |
| 612 | exitcode = None |
| 613 | if os.WIFEXITED(status): |
| 614 | exitcode = os.WEXITSTATUS(status) |
| 615 | elif os.WIFSIGNALED(status): |
| 616 | sig = os.WTERMSIG(status) |
| 617 | try: |
| 618 | sig_name = signal.Signals(sig).name |
| 619 | except ValueError: |
| 620 | sig_name = "signal {}".format(sig) |
| 621 | msg = "Worker (pid:{}) was sent {}!".format( |
| 622 | wpid, sig_name) |
| 623 | |
| 624 | # SIGKILL suggests OOM, log as error |
| 625 | if sig == signal.SIGKILL: |
| 626 | msg += " Perhaps out of memory?" |
| 627 | self.log.error(msg) |
| 628 | elif sig == signal.SIGTERM: |
| 629 | # SIGTERM is expected during graceful shutdown |
| 630 | self.log.info(msg) |
| 631 | else: |
| 632 | # Other signals are unexpected |
| 633 | self.log.warning(msg) |
| 634 | |
| 635 | if exitcode is not None and exitcode != 0: |
| 636 | self.log.error("Worker (pid:%s) exited with code %s.", |
| 637 | wpid, exitcode) |
| 638 | |
| 639 | if exitcode == self.WORKER_BOOT_ERROR: |
| 640 | reason = "Worker failed to boot." |
| 641 | raise HaltServer(reason, self.WORKER_BOOT_ERROR) |
| 642 | if exitcode == self.APP_LOAD_ERROR: |
| 643 | reason = "App failed to load." |
| 644 | raise HaltServer(reason, self.APP_LOAD_ERROR) |
| 645 | |
| 646 | worker = self.WORKERS.pop(wpid, None) |
| 647 | if not worker: |
| 648 | continue |
| 649 | worker.tmp.close() |
| 650 | self.cfg.child_exit(self, worker) |
| 651 | except OSError as e: |
| 652 | if e.errno != errno.ECHILD: |
| 653 | raise |
| 654 |