RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

collect download latencies in 'download_latency' request/response meta key

This commit is contained in:
Pablo Hoffman 2011-05-15 05:24:01 -03:00
parent 668dfcabf3
commit bd8d7f5cf4
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/webclient.py
View File

@ -1,3 +1,4 @@
from time import time
from urlparse import urlparse, urlunparse, urldefrag
from twisted.python import failure
@ -97,7 +98,8 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
self.headers = Headers(request.headers)
self.response_headers = None
self.timeout = request.meta.get('download_timeout') or timeout
self.deferred = defer.Deferred().addCallback(self._build_response)
self.start_time = time()
self.deferred = defer.Deferred().addCallback(self._build_response, request)
self._set_connection_attributes(request)
@ -110,7 +112,8 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
# just in case a broken http/1.1 decides to keep connection alive
self.headers.setdefault("Connection", "close")
def _build_response(self, body):
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time-self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self.url)
@ -125,4 +128,5 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
self.path = self.url
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers