MCPcopy
hub / github.com/scrapy/scrapy / download_request

Method download_request

scrapy/core/downloader/handlers/http11.py:456–484  ·  view source on GitHub ↗
(self, request: Request)

Source from the content-addressed store, hash-verified

454 )
455
456 def download_request(self, request: Request) -> Deferred[Response]:
457 from twisted.internet import reactor
458
459 timeout = request.meta.get("download_timeout") or self._connectTimeout
460 agent = self._get_agent(request, timeout)
461
462 # request details
463 url = urldefrag(request.url)[0]
464 method = to_bytes(request.method)
465 headers = TxHeaders(request.headers)
466 if isinstance(agent, _TunnelingAgent):
467 headers.removeHeader(b"Proxy-Authorization")
468 bodyproducer = _RequestBodyProducer(request.body) if request.body else None
469 start_time = monotonic()
470 d: Deferred[IResponse] = agent.request(
471 method,
472 to_bytes(url, encoding="ascii"),
473 headers,
474 cast("IBodyProducer", bodyproducer),
475 )
476 # set download latency
477 d.addCallback(self._cb_latency, request, start_time)
478 # response body is ready to be consumed
479 d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
480 d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)
481 # check download timeout
482 self._timeout_cl = reactor.callLater(timeout, d3.cancel)
483 d3.addBoth(self._cb_timeout, request, url, timeout)
484 return d3
485
486 def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
487 if self._timeout_cl.active():

Callers 1

download_requestMethod · 0.45

Calls 5

_get_agentMethod · 0.95
to_bytesFunction · 0.90
getMethod · 0.45
requestMethod · 0.45

Tested by

no test coverage detected