diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 97be46f2a..02fa6e287 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -163,7 +163,7 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request, spider, source) +.. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is received for a specific request. This signal might be fired multiple @@ -180,10 +180,6 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object - :param source: a string to identify which handler sent the signal - (current values could be "http11" or "s3") - :type source: :class:`str` object - spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c350cd3c2..bda21b6b9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,9 +33,8 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None, source="http11"): + def __init__(self, settings, crawler=None): self._crawler = crawler - self._source = source from twisted.internet import reactor self._pool = HTTPConnectionPool(reactor, persistent=True) @@ -71,8 +70,8 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler, **kwargs): - return cls(crawler.settings, crawler, **kwargs) + def from_crawler(cls, crawler): + return cls(crawler.settings, crawler) def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -83,7 +82,6 @@ class HTTP11DownloadHandler: warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, - source=self._source, ) return agent.download_request(request) @@ -281,7 +279,7 @@ class ScrapyAgent: _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -291,7 +289,6 @@ class ScrapyAgent: self._fail_on_dataloss = fail_on_dataloss self._txresponse = None self._crawler = crawler - self._source = source def _get_agent(self, request, timeout): from twisted.internet import reactor @@ -430,7 +427,6 @@ class ScrapyAgent: warnsize=warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, - source=self._source, ) ) @@ -468,9 +464,7 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - def __init__( - self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source - ): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -483,7 +477,6 @@ class _ResponseReader(protocol.Protocol): self._bytes_received = 0 self._certificate = None self._crawler = crawler - self._source = source def connectionMade(self): if self._certificate is None: @@ -503,7 +496,6 @@ class _ResponseReader(protocol.Protocol): data=bodyBytes, request=self._request, spider=self._crawler.spider, - source=self._source, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 2366b6394..40a1fa48e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -73,7 +73,6 @@ class S3DownloadHandler: objcls=httpdownloadhandler, settings=settings, crawler=crawler, - source="s3", ) self._download_http = _http_handler.download_request diff --git a/tests/test_engine.py b/tests/test_engine.py index 26f3163cf..acfe94f63 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -112,7 +112,6 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.bytes = defaultdict(lambda: list()) - self.bytes_source = set() self.signals_caught = {} self.spider_class = spider_class @@ -166,9 +165,8 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request, spider, source): + def bytes_received(self, data, request, spider): self.bytes[request].append(data) - self.bytes_source.add(source) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -281,7 +279,6 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) - self.assertEqual(self.run.bytes_source, set(["http11"])) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/":