MCPcopy
hub / github.com/scrapy/scrapy / _TunnelingAgent

Class _TunnelingAgent

scrapy/core/downloader/handlers/http11.py:285–340  ·  view source on GitHub ↗

An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is transparent to the client; thus the agent should behave like there is no

Source from the content-addressed store, hash-verified

283
284
285class _TunnelingAgent(Agent):
286 """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
287 downloads. It may look strange that we have chosen to subclass Agent and not
288 ProxyAgent but consider that after the tunnel is opened the proxy is
289 transparent to the client; thus the agent should behave like there is no
290 proxy involved.
291 """
292
293 def __init__(
294 self,
295 *,
296 reactor: ReactorBase,
297 proxyConf: tuple[str, int, bytes | None],
298 contextFactory: IPolicyForHTTPS,
299 connectTimeout: float | None = None,
300 bindAddress: tuple[str, int] | None = None,
301 pool: HTTPConnectionPool | None = None,
302 ):
303 super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call]
304 self._proxyConf: tuple[str, int, bytes | None] = proxyConf
305 self._contextFactory: IPolicyForHTTPS = contextFactory
306
307 def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:
308 return _TunnelingTCP4ClientEndpoint(
309 reactor=self._reactor,
310 host=uri.host,
311 port=uri.port,
312 proxyConf=self._proxyConf,
313 contextFactory=self._contextFactory,
314 timeout=self._endpointFactory._connectTimeout,
315 bindAddress=self._endpointFactory._bindAddress,
316 )
317
318 def _requestWithEndpoint(
319 self,
320 key: Any,
321 endpoint: TCP4ClientEndpoint,
322 method: bytes,
323 parsedURI: URI,
324 headers: TxHeaders | None,
325 bodyProducer: IBodyProducer | None,
326 requestPath: bytes,
327 ) -> Deferred[IResponse]:
328 # proxy host and port are required for HTTP pool `key`
329 # otherwise, same remote host connection request could reuse
330 # a cached tunneled connection to a different proxy
331 key += self._proxyConf
332 return super()._requestWithEndpoint(
333 key=key,
334 endpoint=endpoint,
335 method=method,
336 parsedURI=parsedURI,
337 headers=headers,
338 bodyProducer=bodyProducer,
339 requestPath=requestPath,
340 )
341
342

Callers 1

_get_agentMethod · 0.85

Calls

no outgoing calls

Tested by

no test coverage detected