| 66 | |
| 67 | |
| 68 | class _ScrapyH2Agent: |
| 69 | def __init__( |
| 70 | self, |
| 71 | context_factory: IPolicyForHTTPS, |
| 72 | pool: H2ConnectionPool, |
| 73 | connect_timeout: int = 10, |
| 74 | bind_address: str | tuple[str, int] | None = None, |
| 75 | crawler: Crawler | None = None, |
| 76 | ) -> None: |
| 77 | self._context_factory = context_factory |
| 78 | self._connect_timeout = connect_timeout |
| 79 | self._bind_address = bind_address |
| 80 | self._pool = pool |
| 81 | self._crawler = crawler |
| 82 | |
| 83 | def _get_agent(self, request: Request, timeout: float | None) -> H2Agent: |
| 84 | from twisted.internet import reactor |
| 85 | |
| 86 | if request.meta.get("proxy"): # pragma: no cover |
| 87 | raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") |
| 88 | bind_address = request.meta.get("bindaddress") or self._bind_address |
| 89 | bind_address = normalize_bind_address(bind_address) |
| 90 | return H2Agent( |
| 91 | reactor=reactor, |
| 92 | context_factory=self._context_factory, |
| 93 | connect_timeout=timeout, |
| 94 | bind_address=bind_address, |
| 95 | pool=self._pool, |
| 96 | ) |
| 97 | |
| 98 | def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: |
| 99 | from twisted.internet import reactor |
| 100 | |
| 101 | timeout = request.meta.get("download_timeout") or self._connect_timeout |
| 102 | agent = self._get_agent(request, timeout) |
| 103 | |
| 104 | start_time = monotonic() |
| 105 | d = agent.request(request, spider) |
| 106 | d.addCallback(self._cb_latency, request, start_time) |
| 107 | |
| 108 | timeout_cl = reactor.callLater(timeout, d.cancel) |
| 109 | d.addBoth(self._cb_timeout, request, timeout, timeout_cl) |
| 110 | return d |
| 111 | |
| 112 | @staticmethod |
| 113 | def _cb_latency( |
| 114 | response: Response, request: Request, start_time: float |
| 115 | ) -> Response: |
| 116 | request.meta["download_latency"] = monotonic() - start_time |
| 117 | return response |
| 118 | |
| 119 | @staticmethod |
| 120 | def _cb_timeout( |
| 121 | response: Response, request: Request, timeout: float, timeout_cl: DelayedCall |
| 122 | ) -> Response: |
| 123 | if timeout_cl.active(): |
| 124 | timeout_cl.cancel() |
| 125 | return response |