From 80925ab845b7f55be97d9bb91015ceee90efc333 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 5 Aug 2019 11:39:07 -0300 Subject: [PATCH 01/36] Get server IP address for HTTP/1.1 responses --- docs/topics/request-response.rst | 12 +++++++++- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/http11.py | 18 ++++++++++----- scrapy/http/response/__init__.py | 5 +++-- tests/test_crawl.py | 27 +++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8997a7f19..a4cc1a7d7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -34,7 +34,7 @@ Request objects :type url: string :param callback: the function that will be called with the response of this - request (once its downloaded) as its first parameter. For more information + request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's :meth:`~scrapy.spiders.Spider.parse` method will be used. @@ -611,6 +611,12 @@ Response objects This represents the :class:`Request` that generated this response. :type request: :class:`Request` object + :param ip_address: The IP address of the server from which the Response originated. + :type ip_address: :class:`ipaddress.IPv4Address` object + + .. FIXME: Add ipaddress.IPv6Address once it's supported + + .. attribute:: Response.url A string containing the URL of the response. @@ -679,6 +685,10 @@ Response objects they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. + .. attribute:: Response.ip_address + + The IP address of the server from which the Response originated. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 157dc3418..11c9dd908 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -172,7 +172,7 @@ class Downloader(object): return response dfd.addCallback(_downloaded) - # 3. After response arrives, remove the request from transferring + # 3. After response arrives, remove the request from transferring # state to free up the transferring slot so it can be used by the # following requests (perhaps those which came from the downloader # middleware itself) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..b690f439f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -4,6 +4,7 @@ import logging import re import warnings from io import BytesIO +from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -382,7 +383,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None + return txresponse, b'', None, None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -418,11 +419,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags = result + txresponse, body, flags, ip_address = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address) @implementer(IBodyProducer) @@ -456,6 +457,11 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._ip_address = None + + def connectionMade(self): + if self._ip_address is None: + self._ip_address = ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -488,16 +494,16 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None)) + self._finished.callback((self._txresponse, body, None, self._ip_address)) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'])) + self._finished.callback((self._txresponse, body, ['partial'], self._ip_address)) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'])) + self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address)) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f92d0901c..ca5ecc02c 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) self._set_url(url) self.request = request self.flags = [] if flags is None else list(flags) + self.ip_address = ip_address @property def meta(self): @@ -76,7 +77,7 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags']: + for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..6281160ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,7 @@ import json import logging +from ipaddress import IPv4Address +from urllib.parse import urlparse from testfixtures import LogCapture from twisted.internet import defer @@ -308,3 +310,28 @@ with multiples lines self.assertIn("[callback] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + + @defer.inlineCallbacks + def test_dns_server_ip_address(self): + from socket import gethostbyname + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = 'https://example.org' + yield crawler.crawl(seed=url) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/status?n=200') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsNone(ip_address) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/echo?body=test') + expected_netloc, _ = urlparse(url).netloc.split(':') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) From e8da7e296691d2b4eb63e2a442bb600e03e5766f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 17:53:39 -0300 Subject: [PATCH 02/36] Test DNS resolution using CrawlerProcess --- tests/CrawlerProcess/ip_address.py | 51 ++++++++++++++++++++++++++++++ tests/test_crawl.py | 10 +----- tests/test_crawler.py | 8 +++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/CrawlerProcess/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py new file mode 100644 index 000000000..6b069cc90 --- /dev/null +++ b/tests/CrawlerProcess/ip_address.py @@ -0,0 +1,51 @@ +from urllib.parse import urlparse + +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver +from twisted.internet.interfaces import IResolverSimple +from zope.interface.declarations import implementer + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerProcess + +from tests.mockserver import MockServer + + +@implementer(IResolverSimple) +class MockThreadedResolver(ThreadedResolver): + """ + Resolves all names to localhost + """ + + @classmethod + def from_crawler(cls, crawler, reactor): + return cls(reactor) + + def install_on_reactor(self,): + self.reactor.installResolver(self) + + def getHostByName(self, name, timeout=None): + return defer.succeed("127.0.0.1") + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mockserver: + settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} + process = CrawlerProcess(settings) + + port = urlparse(mockserver.http_address).port + url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) + process.crawl(LocalhostSpider, url=url) + process.start() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6281160ae..9896058dc 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,7 @@ import json import logging from ipaddress import IPv4Address +from socket import gethostbyname from urllib.parse import urlparse from testfixtures import LogCapture @@ -313,15 +314,6 @@ with multiples lines @defer.inlineCallbacks def test_dns_server_ip_address(self): - from socket import gethostbyname - - crawler = self.runner.create_crawler(SingleRequestSpider) - url = 'https://example.org' - yield crawler.crawl(seed=url) - ip_address = crawler.spider.meta['responses'][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) - crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0ce0674de..dfc1cf448 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + class MySpider(scrapy.Spider): name = 'spider' custom_settings = { @@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + + def test_response_ip_address(self): + log = self.run_script("ip_address.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Host: not.a.real.domain", log) + self.assertIn("Type: ", log) + self.assertIn("IP address: 127.0.0.1", log) From 8529dff41d3d2f6c81ee58c60b16dd9f2b8f72b4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 18:00:56 -0300 Subject: [PATCH 03/36] Update docs regarding Response.ip_address and IPv6 --- docs/topics/request-response.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a4cc1a7d7..17eb63064 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -612,10 +612,7 @@ Response objects :type request: :class:`Request` object :param ip_address: The IP address of the server from which the Response originated. - :type ip_address: :class:`ipaddress.IPv4Address` object - - .. FIXME: Add ipaddress.IPv6Address once it's supported - + :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` .. attribute:: Response.url From a2ae380efcaa5a3419a4f6a35541ae0fb71a2e7f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 13:23:52 -0300 Subject: [PATCH 04/36] Remove unnecessary commas --- scrapy/resolver.py | 2 +- tests/CrawlerProcess/ip_address.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 554a3a14d..f69894b1e 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py index 6b069cc90..949e97172 100644 --- a/tests/CrawlerProcess/ip_address.py +++ b/tests/CrawlerProcess/ip_address.py @@ -21,7 +21,7 @@ class MockThreadedResolver(ThreadedResolver): def from_crawler(cls, crawler, reactor): return cls(reactor) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From bb8f7dc609382153df79774ad9d8f6d33d064279 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:14 -0300 Subject: [PATCH 05/36] Mock DNS server --- tests/mockserver.py | 90 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..585741f1b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,4 @@ +import argparse import json import os import random @@ -6,18 +7,19 @@ from subprocess import Popen, PIPE from urllib.parse import urlencode from OpenSSL import SSL -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from twisted.internet import defer, reactor, ssl +from twisted.internet.task import deferLater +from twisted.names import dns, error +from twisted.names.server import DNSServerFactory +from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource -from twisted.web.server import GzipEncoderFactory -from twisted.web.resource import EncodingResourceWrapper from twisted.web.util import redirectTo -from twisted.internet import reactor, ssl -from twisted.internet.task import deferLater from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 +from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -198,12 +200,10 @@ class Root(Resource): return b'Scrapy mock HTTP server\n' -class MockServer(): +class MockServer: def __enter__(self): - from scrapy.utils.test import get_testenv - - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'], stdout=PIPE, env=get_testenv()) http_address = self.proc.stdout.readline().strip().decode('ascii') https_address = self.proc.stdout.readline().strip().decode('ascii') @@ -224,6 +224,37 @@ class MockServer(): return host + path +class MockDNSResolver: + """ + Implements twisted.internet.interfaces.IResolver partially + """ + + def _resolve(self, name): + record = dns.Record_A(address=b"127.0.0.1") + answer = dns.RRHeader(name=name, payload=record) + return [answer], [], [] + + def query(self, query, timeout=None): + if query.type == dns.A: + return defer.succeed(self._resolve(query.name.name)) + return defer.fail(error.DomainError()) + + def lookupAllRecords(self, name, timeout=None): + return defer.succeed(self._resolve(name)) + + +class MockDNSServer(): + + def __enter__(self): + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + stdout=PIPE, env=get_testenv()) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.proc.kill() + self.proc.communicate() + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), @@ -238,19 +269,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": - root = Root() - factory = Site(root) - httpPort = reactor.listenTCP(0, factory) - contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http") + args = parser.parse_args() - def print_listening(): - httpHost = httpPort.getHost() - httpsHost = httpsPort.getHost() - httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) - httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) - print(httpAddress) - print(httpsAddress) + if args.type == "http": + root = Root() + factory = Site(root) + httpPort = reactor.listenTCP(0, factory) + contextFactory = ssl_context_factory() + httpsPort = reactor.listenSSL(0, factory, contextFactory) + + def print_listening(): + httpHost = httpPort.getHost() + httpsHost = httpsPort.getHost() + httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) + httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + + elif args.type == "dns": + clients = [MockDNSResolver()] + factory = DNSServerFactory(clients=clients) + protocol = dns.DNSDatagramProtocol(controller=factory) + reactor.listenUDP(10053, protocol) + reactor.listenTCP(10053, factory) + + def print_listening(): + print("DNS server running on port 10053") reactor.callWhenRunning(print_listening) reactor.run() From 4851efdfb0885a40a44a2834c6c69d0104326801 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:54 -0300 Subject: [PATCH 06/36] Flake8 adjustments --- tests/mockserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 585741f1b..67139534e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -257,9 +257,9 @@ class MockDNSServer(): def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), - ) + os.path.join(os.path.dirname(__file__), keyfile), + os.path.join(os.path.dirname(__file__), certfile), + ) if cipher_string: ctx = factory.getContext() # disabling TLS1.2+ because it unconditionally enables some strong ciphers From e0ef8ad2d6f958de6ce04cd7756e142efeb1a6a2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 15:52:15 -0300 Subject: [PATCH 07/36] CrawlerRunner test for Response.ip_address --- tests/CrawlerProcess/ip_address.py | 51 ------------------------------ tests/CrawlerRunner/ip_address.py | 37 ++++++++++++++++++++++ tests/mockserver.py | 11 ++++--- tests/test_crawler.py | 20 ++++++++---- 4 files changed, 57 insertions(+), 62 deletions(-) delete mode 100644 tests/CrawlerProcess/ip_address.py create mode 100644 tests/CrawlerRunner/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py deleted file mode 100644 index 949e97172..000000000 --- a/tests/CrawlerProcess/ip_address.py +++ /dev/null @@ -1,51 +0,0 @@ -from urllib.parse import urlparse - -from twisted.internet import defer -from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IResolverSimple -from zope.interface.declarations import implementer - -from scrapy import Spider, Request -from scrapy.crawler import CrawlerProcess - -from tests.mockserver import MockServer - - -@implementer(IResolverSimple) -class MockThreadedResolver(ThreadedResolver): - """ - Resolves all names to localhost - """ - - @classmethod - def from_crawler(cls, crawler, reactor): - return cls(reactor) - - def install_on_reactor(self): - self.reactor.installResolver(self) - - def getHostByName(self, name, timeout=None): - return defer.succeed("127.0.0.1") - - -class LocalhostSpider(Spider): - name = "localhost_spider" - - def start_requests(self): - yield Request(self.url) - - def parse(self, response): - netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) - - -with MockServer() as mockserver: - settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} - process = CrawlerProcess(settings) - - port = urlparse(mockserver.http_address).port - url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) - process.crawl(LocalhostSpider, url=url) - process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py new file mode 100644 index 000000000..5a71536d8 --- /dev/null +++ b/tests/CrawlerRunner/ip_address.py @@ -0,0 +1,37 @@ +from urllib.parse import urlparse + +from twisted.internet import reactor +from twisted.names.client import createResolver + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + +from tests.mockserver import MockServer, MockDNSServer + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) + + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) + + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 67139534e..08a81418c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -246,8 +246,11 @@ class MockDNSResolver: class MockDNSServer(): def __enter__(self): - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) + host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") + self.host = host + self.port = int(port) return self def __exit__(self, exc_type, exc_value, traceback): @@ -292,11 +295,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - reactor.listenUDP(10053, protocol) - reactor.listenTCP(10053, factory) + listener = reactor.listenUDP(0, protocol) def print_listening(): - print("DNS server running on port 10053") + host = listener.getHost() + print("%s:%s" % (host.host, host.port)) reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dfc1cf448..5d381c368 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertNotIn("Asyncio reactor is installed", str(log)) -class CrawlerProcessSubprocess(unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') - +class ScriptRunnerMixin: def run_script(self, script_name): script_path = os.path.join(self.script_dir, script_name) args = (sys.executable, script_path) @@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): stdout, stderr = p.communicate() return stderr.decode('utf-8') + +class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) @@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + +class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + def test_response_ip_address(self): log = self.run_script("ip_address.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Host: not.a.real.domain", log) - self.assertIn("Type: ", log) - self.assertIn("IP address: 127.0.0.1", log) + self.assertIn("INFO: Spider closed (finished)", log) + self.assertIn("INFO: Host: not.a.real.domain", log) + self.assertIn("INFO: Type: ", log) + self.assertIn("INFO: IP address: 127.0.0.1", log) From 13670f0397ba8dcec3dceb1852bad5751406d19d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 16:16:43 -0300 Subject: [PATCH 08/36] Ignore tests/CrawlerRunner directory --- conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index c0de09909..55294feca 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,8 @@ collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess - *_py_files("tests/CrawlerProcess") + *_py_files("tests/CrawlerProcess"), + *_py_files("tests/CrawlerRunner"), ] for line in open('tests/ignores.txt'): From ad70497416527c3d882a64f7803e73155f3fa1da Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 4 Feb 2020 13:30:13 -0300 Subject: [PATCH 09/36] Remove unnecessary parentheses in class definition --- tests/mockserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 08a81418c..30d9bc0e8 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -243,7 +243,7 @@ class MockDNSResolver: return defer.succeed(self._resolve(name)) -class MockDNSServer(): +class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], From 13ba9bc629cb0a77ebaca36a10a0a4984d7cce68 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 12:29:39 -0300 Subject: [PATCH 10/36] Note about Response.ip_address --- docs/topics/request-response.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 17eb63064..89e570028 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,6 +685,8 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. .. method:: Response.copy() From 037ae5b22e6d6600dc537ee5073652ce74e5f47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 10 Feb 2020 19:54:47 +0100 Subject: [PATCH 11/36] =?UTF-8?q?Explicitly=20indicate=20None=20as=20ip=5F?= =?UTF-8?q?address=E2=80=99s=20default=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 89e570028..8f2504a33 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,8 +685,10 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`ip_address` is always ``None``. .. method:: Response.copy() From f85bf77da3c8943f0791dcae893e8294c4d118d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:31:13 -0300 Subject: [PATCH 12/36] Restore unrelated change --- scrapy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..554a3a14d 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self): + def install_on_reactor(self,): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From 889b4718520220d1a81e702ff754ec210a7d3c79 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:40:43 -0300 Subject: [PATCH 13/36] Import changes --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_crawl.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e72275021..190ae1d3b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,11 +1,11 @@ """Download handlers for http and https schemes""" +import ipaddress import logging import re import warnings from contextlib import suppress from io import BytesIO -from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -468,7 +468,7 @@ class _ResponseReader(protocol.Protocol): self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) if self._ip_address is None: - self._ip_address = ip_address(self.transport._producer.getPeer().host) + self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3a9b00ab3..3c110e7a6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -441,13 +441,15 @@ with multiples lines self.assertEqual(cert.getIssuer().commonName, b"localhost") @defer.inlineCallbacks - def test_dns_server_ip_address(self): + def test_dns_server_ip_address_none(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta['responses'][0].ip_address self.assertIsNone(ip_address) + @defer.inlineCallbacks + def test_dns_server_ip_address(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/echo?body=test') expected_netloc, _ = urlparse(url).netloc.split(':') From 91a78eef3ee9de033e66db55c49321b2cc43740e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 8 Mar 2020 22:32:17 -0300 Subject: [PATCH 14/36] Pass callback results as dicts instead of tuples --- scrapy/core/downloader/handlers/http11.py | 56 ++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 190ae1d3b..e904cbc05 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,7 +384,13 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None, None + return { + "txresponse": txresponse, + "body": b"", + "flags": None, + "certificate": None, + "ip_address": None, + } maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -420,12 +426,18 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags, certificate, ip_address = result - status = int(txresponse.code) - headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, - flags=flags, certificate=certificate, ip_address=ip_address) + status = int(result["txresponse"].code) + headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + return respcls( + url=url, + status=status, + headers=headers, + body=result["body"], + flags=result["flags"], + certificate=result["certificate"], + ip_address=result["ip_address"], + ) @implementer(IBodyProducer) @@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback( - (self._txresponse, body, None, self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": None, + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(PotentialDataLoss): - self._finished.callback( - (self._txresponse, body, ['partial'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["partial"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback( - (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["dataloss"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return elif not self._fail_on_dataloss_warned: From 1785095707dec53647c835c0b0861b220e8495af Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 20:41:59 -0300 Subject: [PATCH 15/36] Remove single-use variable --- scrapy/core/downloader/handlers/http11.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e904cbc05..a5b03a62b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -426,12 +426,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - status = int(result["txresponse"].code) headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) return respcls( url=url, - status=status, + status=int(result["txresponse"].code), headers=headers, body=result["body"], flags=result["flags"], From 4270e0a0da66a2cb3a8e904c5ea74f84b7f9d041 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 4 Apr 2020 21:51:02 -0300 Subject: [PATCH 16/36] Fix E731: do not assign a lambda expression --- pytest.ini | 24 +++++++++++----------- scrapy/commands/fetch.py | 4 ++-- scrapy/commands/parse.py | 4 ++-- scrapy/core/downloader/webclient.py | 9 ++++---- scrapy/linkextractors/__init__.py | 10 +++++++-- scrapy/linkextractors/lxmlhtml.py | 6 ++---- tests/test_downloadermiddleware_cookies.py | 7 +++---- tests/test_exporters.py | 11 +++++----- tests/test_pipeline_media.py | 18 ++++++++++------ tests/test_utils_python.py | 4 +++- tests/test_utils_signal.py | 4 +++- 11 files changed, 57 insertions(+), 44 deletions(-) diff --git a/pytest.ini b/pytest.ini index 141a13a4f..4b655b8d5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -35,9 +35,9 @@ flake8-ignore = scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E501 E128 E731 + scrapy/commands/fetch.py E401 E501 E128 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/parse.py E128 E501 E731 + scrapy/commands/parse.py E128 E501 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 @@ -50,12 +50,12 @@ flake8-ignore = scrapy/core/engine.py E501 E128 E127 E502 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E128 W504 - scrapy/core/spidermw.py E501 E731 E126 + scrapy/core/spidermw.py E501 E126 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E241 - scrapy/core/downloader/webclient.py E731 E501 E128 E126 + scrapy/core/downloader/webclient.py E501 E128 E126 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E128 E127 scrapy/core/downloader/handlers/http10.py E501 @@ -90,8 +90,8 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E501 E402 W504 - scrapy/linkextractors/lxmlhtml.py E501 E731 + scrapy/linkextractors/__init__.py E501 E402 W504 + scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader scrapy/loader/__init__.py E501 E128 scrapy/loader/processors.py E501 @@ -184,7 +184,7 @@ flake8-ignore = tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 + tests/test_downloadermiddleware_cookies.py E741 E501 E128 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 @@ -197,7 +197,7 @@ flake8-ignore = tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E501 E741 E128 E124 tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E731 E128 E124 + tests/test_exporters.py E501 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 @@ -207,14 +207,14 @@ flake8-ignore = tests/test_item.py E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E501 E731 E741 E128 E117 E241 + tests/test_loader.py E501 E741 E128 E117 E241 tests/test_logformatter.py E128 E501 E122 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E501 E128 E126 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E731 E128 E502 + tests/test_pipeline_media.py E501 E741 E128 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 @@ -237,11 +237,11 @@ flake8-ignore = tests/test_utils_http.py E501 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E241 tests/test_utils_log.py E741 - tests/test_utils_python.py E501 E731 + tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E731 + tests/test_utils_signal.py E741 F841 tests/test_utils_sitemap.py E128 E501 E124 tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0e149941d..506d1f1b7 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -49,8 +49,8 @@ class Command(ScrapyCommand): def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): raise UsageError() - cb = lambda x: self._print_response(x, opts) - request = Request(args[0], callback=cb, dont_filter=True) + request = Request(args[0], callback=self._print_response, + cb_kwargs={"opts": opts}, dont_filter=True) # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 3ef8ddcb3..d5abe5930 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -147,8 +147,8 @@ class Command(ScrapyCommand): logger.error('Unable to find spider for: %(url)s', {'url': url}) # Request requires callback argument as callable or None, not string - request = Request(url, None) - _start_requests = lambda s: [self.prepare_request(s, request, opts)] + def _start_requests(spider): + yield self.prepare_request(spider, Request(url, None), opts) self.spidercls.start_requests = _start_requests def start_parsing(self, url, opts): diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a71dc5fb3..a90a77b2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. - b = lambda s: to_bytes(s, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = b(path) - host = b(parsed.hostname) + path = to_bytes(path, encoding="ascii") + host = to_bytes(parsed.hostname, encoding="ascii") port = parsed.port - scheme = b(parsed.scheme) - netloc = b(parsed.netloc) + scheme = to_bytes(parsed.scheme, encoding="ascii") + netloc = to_bytes(parsed.netloc, encoding="ascii") if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 6afe867b5..d0b5066b6 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -45,8 +45,14 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) -_matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} + + +def _matches(url, regexs): + return any(r.search(url) for r in regexs) + + +def _is_valid_url(url): + return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} class FilteringLinkExtractor: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index fbac1dc59..ceb37c5f1 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -98,11 +98,9 @@ class LxmlLinkExtractor(FilteringLinkExtractor): unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True, restrict_text=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs lx = LxmlParserLinkExtractor( - tag=tag_func, - attr=attr_func, + tag=lambda x: x in tags, + attr=lambda x: x in attrs, unique=unique, process=process_value, strip=strip, diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 051f66680..a8182e2ef 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -13,10 +13,9 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): - cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1')) - return self.assertEqual( - sorted(cookievaleq(first)), - sorted(cookievaleq(second)), msg) + def split_cookies(cookies): + return sorted(re.split(r";\s*", cookies.decode("latin1"))) + return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) def setUp(self): self.spider = Spider('foo') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6e2507508..160912847 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -215,11 +215,12 @@ class CsvItemExporterTest(BaseItemExporterTest): return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): - first = to_unicode(first) - second = to_unicode(second) - csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) - for line in csv.splitlines(True)] - return self.assertEqual(csvsplit(first), csvsplit(second), msg) + def split_csv(csv): + return [ + sorted(re.split(r"(,|\s+)", line)) + for line in to_unicode(csv).splitlines(True) + ] + return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d369e147d..f84f47816 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -199,12 +199,19 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): pipeline_class = MockedMediaPipeline + def _callback(self, result): + self.pipe._mockcalled.append('request_callback') + return result + + def _errback(self, result): + self.pipe._mockcalled.append('request_errback') + return result + @inlineCallbacks def test_result_succeed(self): - cb = lambda _: self.pipe._mockcalled.append('request_callback') or _ - eb = lambda _: self.pipe._mockcalled.append('request_errback') or _ rsp = Response('http://url1') - req = Request('http://url1', meta=dict(response=rsp), callback=cb, errback=eb) + req = Request('http://url1', meta=dict(response=rsp), + callback=self._callback, errback=self._errback) item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(True, rsp)]) @@ -215,10 +222,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_result_failure(self): self.pipe.LOG_FAILED_RESULTS = False - cb = lambda _: self.pipe._mockcalled.append('request_callback') or _ - eb = lambda _: self.pipe._mockcalled.append('request_errback') or _ fail = Failure(Exception()) - req = Request('http://url1', meta=dict(response=fail), callback=cb, errback=eb) + req = Request('http://url1', meta=dict(response=fail), + callback=self._callback, errback=self._errback) item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(False, fail)]) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 8cb8df15b..65e6ba876 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -145,7 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase): get_z = operator.itemgetter('z') get_meta = operator.attrgetter('meta') - compare_z = lambda obj: get_z(get_meta(obj)) + + def compare_z(obj): + return get_z(get_meta(obj)) self.assertTrue(equal_attributes(a, b, [compare_z, 'x'])) # fail z equality diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 9f6da09ed..bb211dc60 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -90,8 +90,10 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): + def test_handler(): + return defer.Deferred() + test_signal = object() - test_handler = lambda: defer.Deferred() dispatcher.connect(test_handler, test_signal) with LogCapture() as l: send_catch_log(test_signal) From c887fe37adfe529ed2afabd2d08c3aac00a819c0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 4 Apr 2020 22:15:36 -0300 Subject: [PATCH 17/36] Simplify parse command --- scrapy/commands/parse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index d5abe5930..1cefed106 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -146,9 +146,8 @@ class Command(ScrapyCommand): if not self.spidercls: logger.error('Unable to find spider for: %(url)s', {'url': url}) - # Request requires callback argument as callable or None, not string def _start_requests(spider): - yield self.prepare_request(spider, Request(url, None), opts) + yield self.prepare_request(spider, Request(url), opts) self.spidercls.start_requests = _start_requests def start_parsing(self, url, opts): From 47a992615a046d20c31bfca6f8b65c3194e9fd30 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 15 Apr 2020 19:57:34 -0300 Subject: [PATCH 18/36] serialize requests with callback references as spider attribute You could define a spider attribute that references a callback method but if this method has a different name than your spider attribute, the request serializer is not able to find it on the spider class. With this commit we're fixing this behavior as we're searching for callback references in the spider object itself instead of looking for attributes with the same function's name, that could be different. --- scrapy/utils/reqser.py | 12 ++++++++---- tests/test_utils_reqser.py | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 749bbc387..78e13ec10 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,6 +1,8 @@ """ Helper functions for serializing (and deserializing) requests. """ +import inspect + from scrapy.http import Request from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -90,10 +92,12 @@ def _find_method(obj, func): pass else: if func_self is obj: - name = func.__func__.__name__ - if _is_private_method(name): - return _mangle_private_name(obj, func, name) - return name + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + if obj_func.__func__ is func.__func__: + if _is_private_method(name): + return _mangle_private_name(obj, func, name) + return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c7572f02c..cf84f8fbd 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_reference_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.parse_item_reference, + errback=self.spider.handle_error_reference) + self._assert_serializes_ok(r, spider=self.spider) + request_dict = request_to_dict(r, self.spider) + self.assertEqual(request_dict['callback'], 'parse_item_reference') + self.assertEqual(request_dict['errback'], 'handle_error_reference') + + def test_private_reference_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider._TestSpider__parse_item_reference, + errback=self.spider._TestSpider__handle_error_reference) + self._assert_serializes_ok(r, spider=self.spider) + request_dict = request_to_dict(r, self.spider) + self.assertEqual(request_dict['callback'], + '_TestSpider__parse_item_reference') + self.assertEqual(request_dict['errback'], + '_TestSpider__handle_error_reference') + def test_private_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider._TestSpider__parse_item_private, @@ -131,8 +151,28 @@ class TestSpiderMixin: pass +def parse_item(response): + pass + + +def handle_error(failure): + pass + + +def private_parse_item(response): + pass + + +def private_handle_error(failure): + pass + + class TestSpider(Spider, TestSpiderMixin): name = 'test' + parse_item_reference = parse_item + handle_error_reference = handle_error + __parse_item_reference = private_parse_item + __handle_error_reference = private_handle_error def parse_item(self, response): pass From 901892dab380d54186ae855bf65a20eb2467de04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 14:48:38 +0200 Subject: [PATCH 19/36] Fix the hoverxref configuration --- docs/conf.py | 2 -- tox.ini | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4414ef637..813417bae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -295,8 +295,6 @@ intersphinx_mapping = { # ------------------------------------ hoverxref_auto_ref = True -hoverxref_project = "scrapy" -hoverxref_version = release hoverxref_role_types = { "class": "tooltip", "confval": "tooltip", diff --git a/tox.ini b/tox.ini index b1babc7fd..cd118c921 100644 --- a/tox.ini +++ b/tox.ini @@ -74,11 +74,15 @@ deps = changedir = docs deps = -rdocs/requirements.txt +setenv = + READTHEDOCS_PROJECT=scrapy + READTHEDOCS_VERSION=master [testenv:docs] basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} +setenv = {[docs]setenv} commands = sphinx-build -W -b html . {envtmpdir}/html @@ -86,6 +90,7 @@ commands = basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} +setenv = {[docs]setenv} commands = sphinx-build -b coverage . {envtmpdir}/coverage @@ -93,6 +98,7 @@ commands = basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} +setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck From e0921cab667a1fefe9730363ed45091edf1250e8 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 16 Apr 2020 11:18:56 -0300 Subject: [PATCH 20/36] remove not used code This code is not needed anymore because we're getting the already mangled name when matching func with spider attributes. --- scrapy/utils/reqser.py | 16 ---------------- tests/test_utils_reqser.py | 37 +------------------------------------ 2 files changed, 1 insertion(+), 52 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 78e13ec10..1392b2c61 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -70,20 +70,6 @@ def request_from_dict(d, spider=None): ) -def _is_private_method(name): - return name.startswith('__') and not name.endswith('__') - - -def _mangle_private_name(obj, func, name): - qualname = getattr(func, '__qualname__', None) - if qualname is None: - classname = obj.__class__.__name__.lstrip('_') - return '_%s%s' % (classname, name) - else: - splits = qualname.split('.') - return '_%s%s' % (splits[-2], splits[-1]) - - def _find_method(obj, func): if obj: try: @@ -95,8 +81,6 @@ def _find_method(obj, func): members = inspect.getmembers(obj, predicate=inspect.ismethod) for name, obj_func in members: if obj_func.__func__ is func.__func__: - if _is_private_method(name): - return _mangle_private_name(obj, func, name) return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index cf84f8fbd..47853d812 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -2,7 +2,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name +from scrapy.utils.reqser import request_to_dict, request_from_dict class RequestSerializationTest(unittest.TestCase): @@ -101,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) - def test_private_callback_name_matching(self): - self.assertTrue(_is_private_method('__a')) - self.assertTrue(_is_private_method('__a_')) - self.assertTrue(_is_private_method('__a_a')) - self.assertTrue(_is_private_method('__a_a_')) - self.assertTrue(_is_private_method('__a__a')) - self.assertTrue(_is_private_method('__a__a_')) - self.assertTrue(_is_private_method('__a___a')) - self.assertTrue(_is_private_method('__a___a_')) - self.assertTrue(_is_private_method('___a')) - self.assertTrue(_is_private_method('___a_')) - self.assertTrue(_is_private_method('___a_a')) - self.assertTrue(_is_private_method('___a_a_')) - self.assertTrue(_is_private_method('____a_a_')) - - self.assertFalse(_is_private_method('_a')) - self.assertFalse(_is_private_method('_a_')) - self.assertFalse(_is_private_method('__a__')) - self.assertFalse(_is_private_method('__')) - self.assertFalse(_is_private_method('___')) - self.assertFalse(_is_private_method('____')) - - def _assert_mangles_to(self, obj, name): - func = getattr(obj, name) - self.assertEqual( - _mangle_private_name(obj, func, func.__name__), - name - ) - - def test_private_name_mangling(self): - self._assert_mangles_to( - self.spider, '_TestSpider__parse_item_private') - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') - def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) From 94c95020b391c3298f4a7fd7608d48d74117bf43 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 16 Apr 2020 11:37:03 -0300 Subject: [PATCH 21/36] add comment to explain the use of __func__ instead of instance method objects --- scrapy/utils/reqser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 1392b2c61..5ea2aafb8 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -80,6 +80,13 @@ def _find_method(obj, func): if func_self is obj: members = inspect.getmembers(obj, predicate=inspect.ismethod) for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html if obj_func.__func__ is func.__func__: return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) From c9229922772a4d7f92a26786d6ea441609043a09 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 16 Apr 2020 11:37:37 -0300 Subject: [PATCH 22/36] Tests: Move code inside __main__ block --- tests/CrawlerRunner/ip_address.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5a71536d8..826374cd4 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -23,15 +23,16 @@ class LocalhostSpider(Spider): self.logger.info("IP address: %s" % response.ip_address) -with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: - port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) +if __name__ == "__main__": + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) - servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) - configure_logging() - runner = CrawlerRunner() - d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) - reactor.run() + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run() From 1ade3fc723d1e5d7b6a3300b454d32656bdb8d28 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 17 Apr 2020 10:34:34 -0300 Subject: [PATCH 23/36] trying to improve test coverage --- tests/test_utils_reqser.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 47853d812..50b026d1c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -110,6 +110,21 @@ class RequestSerializationTest(unittest.TestCase): r = Request("http://www.example.com", callback=self.spider.parse_item) self.assertRaises(ValueError, request_to_dict, r) + def test_unserializable_callback3(self): + """Parser method is removed or replaced dynamically.""" + + class MySpider(Spider): + + name = 'my_spider' + + def parse(self, response): + pass + + spider = MySpider() + r = Request("http://www.example.com", callback=spider.parse) + setattr(spider, 'parse', None) + self.assertRaises(ValueError, request_to_dict, r, spider=spider) + class TestSpiderMixin: def __mixin_callback(self, response): From 04b6295a69174e81beceb0b1429fa3775949e99d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 17 Apr 2020 20:50:17 -0300 Subject: [PATCH 24/36] Docs: replace deprecated FEED_* settings --- docs/topics/practices.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index e3e8fdc72..cf1de1bd1 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it. ... process = CrawlerProcess(settings={ - 'FEED_FORMAT': 'json', - 'FEED_URI': 'items.json' + "FEEDS": { + "items.json": {"format": "json"}, + }, }) process.crawl(MySpider) From bfeb2c8c13de0c45af21228f69395a1131913da5 Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Sat, 18 Apr 2020 20:51:26 +0530 Subject: [PATCH 25/36] Added warning to use double quotes in Windows for scrapy shell in shell.rst (#4450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * modified debugging memory leaks with guppy in leaks.rst * modified leaks.rst(issue #4285) * removed guppy from telnet.py * Fix undefined name error * removed hpy key from telnet_vars in telnet.py * updated shell.rst * Update docs/topics/shell.rst Co-Authored-By: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/topics/shell.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8f7518b19..0f46f1c87 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -156,6 +156,17 @@ First, we launch the shell:: scrapy shell 'https://scrapy.org' --nolog +.. note:: + + Remember to always enclose URLs in quotes when running the Scrapy shell from + the command line, otherwise URLs containing arguments (i.e. the ``&`` character) + will not work. + + On Windows, use double quotes instead:: + + scrapy shell "https://scrapy.org" --nolog + + Then, the shell fetches the URL (using the Scrapy downloader) and prints the list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: From e4750f2fbdacbeb7a20ae7c6b13bba3fb0f7ad54 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Mon, 20 Apr 2020 21:17:57 +0530 Subject: [PATCH 26/36] async/deferred signal handlers (#4390) * [docs] async/deferred signal handlers * [docs] update deferred signals example * [docs] add subsections for built-in signals * docs(signals): update signal handler example * docs(signals): update signal handler example --- docs/topics/signals.rst | 96 +++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 2def53848..8661f86a0 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,8 +16,7 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. -Here is a simple example showing how you can catch signals and perform some action: -:: +Here is a simple example showing how you can catch signals and perform some action:: from scrapy import signals from scrapy import Spider @@ -52,9 +51,45 @@ Deferred signal handlers ======================== Some signals support returning :class:`~twisted.internet.defer.Deferred` -objects from their handlers, see the :ref:`topics-signals-ref` below to know -which ones. +objects from their handlers, allowing you to run asynchronous code that +does not block Scrapy. If a signal handler returns a +:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that +:class:`~twisted.internet.defer.Deferred` to fire. +Let's take an example:: + + class SignalSpider(scrapy.Spider): + name = 'signals' + start_urls = ['http://quotes.toscrape.com/page/1/'] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped) + return spider + + def item_scraped(self, item): + # Send the scraped item to the server + d = treq.post( + 'http://example.com/post', + json.dumps(item).encode('ascii'), + headers={b'Content-Type': [b'application/json']} + ) + + # The next item will be scraped only after + # deferred (d) is fired + return d + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), + } + +See the :ref:`topics-signals-ref` below to know which signals support +:class:`~twisted.internet.defer.Deferred`. .. _topics-signals-ref: @@ -66,9 +101,12 @@ Built-in signals reference Here's the list of Scrapy built-in signals and their meaning. -engine_started +Engine signals -------------- +engine_started +~~~~~~~~~~~~~~ + .. signal:: engine_started .. function:: engine_started() @@ -81,7 +119,7 @@ engine_started getting fired before :signal:`spider_opened`. engine_stopped --------------- +~~~~~~~~~~~~~~ .. signal:: engine_stopped .. function:: engine_stopped() @@ -91,9 +129,20 @@ engine_stopped This signal supports returning deferreds from their handlers. -item_scraped +Item signals ------------ +.. note:: + As at max :setting:`CONCURRENT_ITEMS` items are processed in + parallel, many deferreds are fired together using + :class:`~twisted.internet.defer.DeferredList`. Hence the next + batch waits for the :class:`~twisted.internet.defer.DeferredList` + to fire and then runs the respective item signal handler for + the next batch of scraped items. + +item_scraped +~~~~~~~~~~~~ + .. signal:: item_scraped .. function:: item_scraped(item, response, spider) @@ -112,7 +161,7 @@ item_scraped :type response: :class:`~scrapy.http.Response` object item_dropped ------------- +~~~~~~~~~~~~ .. signal:: item_dropped .. function:: item_dropped(item, response, exception, spider) @@ -137,7 +186,7 @@ item_dropped :type exception: :exc:`~scrapy.exceptions.DropItem` exception item_error ------------- +~~~~~~~~~~ .. signal:: item_error .. function:: item_error(item, response, spider, failure) @@ -159,8 +208,11 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure +Spider signals +-------------- + spider_closed -------------- +~~~~~~~~~~~~~ .. signal:: spider_closed .. function:: spider_closed(spider, reason) @@ -183,7 +235,7 @@ spider_closed :type reason: str spider_opened -------------- +~~~~~~~~~~~~~ .. signal:: spider_opened .. function:: spider_opened(spider) @@ -198,7 +250,7 @@ spider_opened :type spider: :class:`~scrapy.spiders.Spider` object spider_idle ------------ +~~~~~~~~~~~ .. signal:: spider_idle .. function:: spider_idle(spider) @@ -228,7 +280,7 @@ spider_idle due to duplication). spider_error ------------- +~~~~~~~~~~~~ .. signal:: spider_error .. function:: spider_error(failure, response, spider) @@ -246,8 +298,11 @@ spider_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.spiders.Spider` object +Request signals +--------------- + request_scheduled ------------------ +~~~~~~~~~~~~~~~~~ .. signal:: request_scheduled .. function:: request_scheduled(request, spider) @@ -264,7 +319,7 @@ request_scheduled :type spider: :class:`~scrapy.spiders.Spider` object request_dropped ---------------- +~~~~~~~~~~~~~~~ .. signal:: request_dropped .. function:: request_dropped(request, spider) @@ -281,7 +336,7 @@ request_dropped :type spider: :class:`~scrapy.spiders.Spider` object request_reached_downloader ---------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) @@ -297,7 +352,7 @@ request_reached_downloader :type spider: :class:`~scrapy.spiders.Spider` object request_left_downloader ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) @@ -315,8 +370,11 @@ request_left_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +Response signals +---------------- + response_received ------------------ +~~~~~~~~~~~~~~~~~ .. signal:: response_received .. function:: response_received(response, request, spider) @@ -336,7 +394,7 @@ response_received :type spider: :class:`~scrapy.spiders.Spider` object response_downloaded -------------------- +~~~~~~~~~~~~~~~~~~~ .. signal:: response_downloaded .. function:: response_downloaded(response, request, spider) From efb6f13debf9406a214a9cee3d94d47875d542f5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 23 Apr 2020 07:40:10 -0300 Subject: [PATCH 27/36] Remove assertions from production code (#4440) --- scrapy/commands/__init__.py | 3 ++- scrapy/contracts/default.py | 6 +++++- scrapy/core/downloader/middleware.py | 5 +++-- scrapy/core/engine.py | 26 ++++++++++++++++++-------- scrapy/core/scraper.py | 6 +++++- scrapy/crawler.py | 3 ++- scrapy/http/request/__init__.py | 3 ++- scrapy/pipelines/files.py | 6 ++++-- scrapy/utils/iterators.py | 10 ++++++---- scrapy/utils/reactor.py | 3 ++- tests/test_utils_iterators.py | 2 +- 11 files changed, 50 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index a573a03d9..9f8e6986a 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,7 +23,8 @@ class ScrapyCommand: self.settings = None # set in scrapy.cmdline def set_crawler(self, crawler): - assert not hasattr(self, '_crawler'), "crawler already set" + if hasattr(self, '_crawler'): + raise RuntimeError("crawler already set") self._crawler = crawler def syntax(self): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 3002fc702..a1b0f8f22 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -58,7 +58,11 @@ class ReturnsContract(Contract): def __init__(self, *args, **kwargs): super(ReturnsContract, self).__init__(*args, **kwargs) - assert len(self.args) in [1, 2, 3] + if len(self.args) not in [1, 2, 3]: + raise ValueError( + "Incorrect argument quantity: expected 1, 2 or 3, got %i" + % len(self.args) + ) self.obj_name = self.args[0] or None self.obj_type = self.objects[self.obj_name] diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 5a03dcdf7..4c2eea522 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -45,8 +45,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_response(response): - assert response is not None, 'Received None in process_response' - if isinstance(response, Request): + if response is None: + raise TypeError("Received None in process_response") + elif isinstance(response, Request): return response for method in self.methods['process_response']: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 66cf9ad9a..77d71846e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -73,7 +73,8 @@ class ExecutionEngine: @defer.inlineCallbacks def start(self): """Start the execution engine""" - assert not self.running, "Engine already running" + if self.running: + raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True @@ -82,7 +83,8 @@ class ExecutionEngine: def stop(self): """Stop the execution engine gracefully""" - assert self.running, "Engine not running" + if not self.running: + raise RuntimeError("Engine not running") self.running = False dfd = self._close_all_spiders() return dfd.addBoth(lambda _: self._finish_stopping_engine()) @@ -165,7 +167,11 @@ class ExecutionEngine: return d def _handle_downloader_output(self, response, request, spider): - assert isinstance(response, (Request, Response, Failure)), response + if not isinstance(response, (Request, Response, Failure)): + raise TypeError( + "Incorrect type: expected Request, Response or Failure, got %s: %r" + % (type(response), response) + ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): self.crawl(response, spider) @@ -205,8 +211,8 @@ class ExecutionEngine: return not bool(self.slot) def crawl(self, request, spider): - assert spider in self.open_spiders, \ - "Spider %r not opened when crawling: %s" % (spider.name, request) + if spider not in self.open_spiders: + raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) self.schedule(request, spider) self.slot.nextcall.schedule() @@ -232,7 +238,11 @@ class ExecutionEngine: slot.add_request(request) def _on_success(response): - assert isinstance(response, (Response, Request)) + if not isinstance(response, (Response, Request)): + raise TypeError( + "Incorrect type: expected Response or Request, got %s: %r" + % (type(response), response) + ) if isinstance(response, Response): response.request = request # tie request to response received logkws = self.logformatter.crawled(request, response, spider) @@ -253,8 +263,8 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): - assert self.has_capacity(), "No free spider slot when opening %r" % \ - spider.name + if not self.has_capacity(): + raise RuntimeError("No free spider slot when opening %r" % spider.name) logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3e4826216..edbb4dd66 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -123,7 +123,11 @@ class Scraper: def _scrape(self, response, request, spider): """Handle the downloaded response or failure through the spider callback/errback""" - assert isinstance(response, (Response, Failure)) + if not isinstance(response, (Response, Failure)): + raise TypeError( + "Incorrect type: expected Response or Failure, got %s: %r" + % (type(response), response) + ) dfd = self._scrape2(response, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, response, spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 20990ea41..6f43771e2 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -78,7 +78,8 @@ class Crawler: @defer.inlineCallbacks def crawl(self, *args, **kwargs): - assert not self.crawling, "Crawling already taking place" + if self.crawling: + raise RuntimeError("Crawling already taking place") self.crawling = True try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0a6637af8..a98ba9960 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -24,7 +24,8 @@ class Request(object_ref): self.method = str(method).upper() self._set_url(url) self._set_body(body) - assert isinstance(priority, int), "Request priority not an integer: %r" % priority + if not isinstance(priority, int): + raise TypeError("Request priority not an integer: %r" % priority) self.priority = priority if callback is not None and not callable(callback): diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index aab645d3d..ae365db5b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -106,7 +106,8 @@ class S3FilesStore: else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection - assert uri.startswith('s3://') + if not uri.startswith("s3://"): + raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -266,7 +267,8 @@ class FTPFilesStore: USE_ACTIVE_MODE = None def __init__(self, uri): - assert uri.startswith('ftp://') + if not uri.startswith("ftp://"): + raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) u = urlparse(uri) self.port = u.port self.host = u.hostname diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b71419111..5e15bf0c8 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): expected_types = (Response, str, bytes) - assert isinstance(obj, expected_types), \ - "obj must be %s, not %s" % ( - " or ".join(t.__name__ for t in expected_types), - type(obj).__name__) + if not isinstance(obj, expected_types): + expected_types_str = " or ".join(t.__name__ for t in expected_types) + raise TypeError( + "Object %r must be %s, not %s" + % (obj, expected_types_str, type(obj).__name__) + ) if isinstance(obj, Response): if not unicode: return obj.body diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5308812d6..3c705f69b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor - assert len(portrange) <= 2, "invalid portrange: %s" % portrange + if len(portrange) > 2: + raise ValueError("invalid portrange: %s" % portrange) if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ec8311298..a85087619 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -157,7 +157,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_objtype_exception(self): i = self.xmliter(42, 'product') - self.assertRaises(AssertionError, next, i) + self.assertRaises(TypeError, next, i) def test_xmliter_encoding(self): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' From ffe576c4ed192882d1e40fef815f0c1d5354249a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Apr 2020 11:44:36 +0200 Subject: [PATCH 28/36] Cover Scrapy 2.1 in the release notes (#4499) Co-authored-by: Mikhail Korobov --- docs/news.rst | 147 +++++++++++++++++++++++++++++++ docs/topics/request-response.rst | 5 ++ 2 files changed, 152 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index e9b7140cd..a158246eb 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,153 @@ Release notes ============= +.. _release-2.1.0: + +Scrapy 2.1.0 (2020-04-24) +------------------------- + +Highlights: + +* New :setting:`FEEDS` setting to export to multiple feeds +* New :attr:`Response.ip_address ` attribute + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :exc:`AssertionError` exceptions triggered by :ref:`assert ` + statements have been replaced by new exception types, to support running + Python in optimized mode (see :option:`-O`) without changing Scrapy’s + behavior in any unexpected ways. + + If you catch an :exc:`AssertionError` exception from Scrapy, update your + code to catch the corresponding new exception. + + (:issue:`4440`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use + :setting:`SCHEDULER_DEBUG` instead (:issue:`4385`) + +* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use + :setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`) + +* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` + middleware has been removed, including the entire + :class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers + work out of the box (:issue:`4431`) + +* The ``spiders`` property has been removed from + :class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader + ` or instantiate + :setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`) + +* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode`` + classes have been removed from :mod:`scrapy.utils.datatypes` + (:issue:`4400`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in + favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`, + :issue:`4507`) + + +New features +~~~~~~~~~~~~ + +* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds + with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support multiple + ``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support + specifying an output format by appending ``:`` to the output file + (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The new :attr:`Response.ip_address ` + attribute gives access to the IP address that originated a response + (:issue:`3903`, :issue:`3940`) + +* A warning is now issued when a value in + :attr:`~scrapy.spiders.Spider.allowed_domains` includes a port + (:issue:`50`, :issue:`3198`, :issue:`4413`) + +* Zsh completion now excludes used option aliases from the completion list + (:issue:`4438`) + + +Bug fixes +~~~~~~~~~ + +* :ref:`Request serialization ` no longer breaks for + callbacks that are spider attributes which are assigned a function with a + different name (:issue:`4500`) + +* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer + cause a :exc:`TypeError` exception (:issue:`4410`) + +* Zsh completion no longer allows options after arguments (:issue:`4438`) + +* zope.interface 5.0.0 and later versions are now supported + (:issue:`4447`, :issue:`4448`) + +* :meth:`Spider.make_requests_from_url + `, deprecated in Scrapy + 1.4.0, now issues a warning when used (:issue:`4412`) + + +Documentation +~~~~~~~~~~~~~ + +* Improved the documentation about signals that allow their handlers to + return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`, + :issue:`4390`) + +* Our PyPI entry now includes links for our documentation, our source code + repository and our issue tracker (:issue:`4456`) + +* Covered the `curl2scrapy `_ + service in the documentation (:issue:`4206`, :issue:`4455`) + +* Removed references to the Guppy library, which only works in Python 2 + (:issue:`4285`, :issue:`4343`) + +* Extended use of InterSphinx to link to Python 3 documentation + (:issue:`4444`, :issue:`4445`) + +* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`, + :issue:`4496`, :issue:`4503`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Removed warnings about using old, removed settings (:issue:`4404`) + +* Removed a warning about importing + :class:`~twisted.internet.testing.StringTransport` from + ``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`) + +* Removed outdated Debian package build files (:issue:`4384`) + +* Removed :class:`object` usage as a base class (:issue:`4430`) + +* Removed code that added support for old versions of Twisted that we no + longer support (:issue:`4472`) + +* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`, + :issue:`4481`) + +* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`, + :issue:`4446`, :issue:`4489`) + + .. _release-2.0.1: Scrapy 2.0.1 (2020-03-18) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 5eb4915cd..024f46466 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -619,6 +619,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + .. versionadded:: 2.1.0 + The ``ip_address`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -710,6 +713,8 @@ Response objects .. attribute:: Response.ip_address + .. versionadded:: 2.1.0 + The IP address of the server from which the Response originated. This attribute is currently only populated by the HTTP 1.1 download From 3878b67a3771102d4b6668ac749afbec7dc85a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Apr 2020 11:46:54 +0200 Subject: [PATCH 29/36] =?UTF-8?q?Bump=20version:=202.0.0=20=E2=86=92=202.1?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f347a0cd0..de22a2783 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0 +current_version = 2.1.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 227cea215..7ec1d6db4 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 From c207dbf939811176a7b094e0f2547aa7846b1cf8 Mon Sep 17 00:00:00 2001 From: Ashe Date: Tue, 28 Apr 2020 02:45:19 +0900 Subject: [PATCH 30/36] Remove the asyncio warning from coroutines page (#4513) --- docs/topics/coroutines.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 5f61d6796..7a9ecd4d5 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -7,10 +7,6 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related API and behavior changes without a - deprecation period or warning. - .. _coroutine-support: Supported callables From e3c3ec2ba988f654be1676586714fd96dba32c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 13:48:50 +0200 Subject: [PATCH 31/36] Run quick tests first in Travis CI --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..dc91dfe4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,9 @@ matrix: python: 3.8 - env: TOXENV=flake8 python: 3.8 + - env: TOXENV=docs + python: 3.7 # Keep in sync with .readthedocs.yml + - env: TOXENV=pypy3 - env: TOXENV=py35 python: 3.5 @@ -28,8 +31,6 @@ matrix: python: 3.8 - env: TOXENV=py38-asyncio python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml install: - | if [ "$TOXENV" = "pypy3" ]; then From 5c0f11b4ef1d58de4245d9f4ac9a26f21faf082c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 17:32:53 +0200 Subject: [PATCH 32/36] Simplify the asyncio Tox environment --- .travis.yml | 4 ++-- tox.ini | 12 +----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..a924eb68c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ matrix: python: 3.5 - env: TOXENV=pinned python: 3.5 - - env: TOXENV=py35-asyncio + - env: TOXENV=asyncio python: 3.5.2 - env: TOXENV=py36 python: 3.6 @@ -26,7 +26,7 @@ matrix: python: 3.8 - env: TOXENV=extra-deps python: 3.8 - - env: TOXENV=py38-asyncio + - env: TOXENV=asyncio python: 3.8 - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml diff --git a/tox.ini b/tox.ini index cd118c921..697328ebd 100644 --- a/tox.ini +++ b/tox.ini @@ -102,16 +102,6 @@ setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck -[asyncio] +[testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio - -[testenv:py35-asyncio] -basepython = python3.5 -deps = {[testenv]deps} -commands = {[asyncio]commands} - -[testenv:py38-asyncio] -basepython = python3.8 -deps = {[testenv]deps} -commands = {[asyncio]commands} From 3a64f3eb2902ed8168b78c43f3516cf657873cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 28 Apr 2020 17:44:19 +0200 Subject: [PATCH 33/36] Remove TOXENV from .travis.yml unless needed --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e1a9617..b029d8bda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,17 +12,14 @@ matrix: - env: TOXENV=flake8 python: 3.8 - env: TOXENV=pypy3 - - env: TOXENV=py35 - python: 3.5 + - python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=py35-asyncio python: 3.5.2 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 + - python: 3.6 + - python: 3.7 + - env: PYPI_RELEASE_JOB=true python: 3.8 - env: TOXENV=extra-deps python: 3.8 @@ -62,4 +59,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" + condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" From 83d7360bb709cf2c73680260c58b767006f42b12 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 4 May 2020 02:00:11 +0500 Subject: [PATCH 34/36] Don't mention unsupported package versions in docs --- docs/topics/settings.rst | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 18f81838f..e3da1bd12 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``). .. note:: HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, - unless you use Twisted<11.1, or if you really want to use HTTP/1.0 - and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme - accordingly, i.e. to - ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. + unless you really want to use HTTP/1.0 and override + :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly, + i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY @@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things). Scrapy also has another context factory class that you can set, ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, which uses the platform's certificates to validate remote endpoints. - **This is only available if you use Twisted>=14.0.** If you do use a custom ContextFactory, make sure its ``__init__`` method accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping @@ -494,10 +492,6 @@ This setting must be one of these string values: - ``'TLSv1.2'``: forces TLS version 1.2 - ``'SSLv3'``: forces SSL version 3 (**not recommended**) -.. note:: - - We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 - or above (Twisted>=14.0 if you can). .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -660,8 +654,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_maxsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_WARNSIZE DOWNLOAD_WARNSIZE @@ -679,8 +671,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_warnsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_FAIL_ON_DATALOSS DOWNLOAD_FAIL_ON_DATALOSS From fe6154e4faee375e7f47d61ceafabde7a3289bf3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 4 May 2020 18:18:38 +0500 Subject: [PATCH 35/36] clarify DOWNLOADER_HTTPCLIENTFACTORY docs --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e3da1bd12..f06d9db3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -421,7 +421,7 @@ connections (for ``HTTP10DownloadHandler``). HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override - :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme accordingly, + :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY From 17c0cf64aee1641e1ad33c5b46a61435c5969f2f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 5 May 2020 19:14:48 -0300 Subject: [PATCH 36/36] Flake8: remove W504 code (#4525) Co-authored-by: Mikhail Korobov --- pytest.ini | 16 ++++++++-------- scrapy/contracts/__init__.py | 4 ++-- scrapy/downloadermiddlewares/redirect.py | 11 +++++++---- scrapy/extensions/telnet.py | 6 ++++-- scrapy/linkextractors/__init__.py | 3 +-- scrapy/spidermiddlewares/referer.py | 14 ++++++++------ scrapy/utils/gz.py | 3 +-- tests/test_utils_http.py | 8 ++++---- 8 files changed, 35 insertions(+), 30 deletions(-) diff --git a/pytest.ini b/pytest.ini index e8911ee3f..4f3494e0e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -44,12 +44,12 @@ flake8-ignore = scrapy/commands/startproject.py E127 E501 E128 scrapy/commands/version.py E501 E128 # scrapy/contracts - scrapy/contracts/__init__.py E501 W504 + scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core scrapy/core/engine.py E501 E128 E127 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 W504 + scrapy/core/scraper.py E501 E128 scrapy/core/spidermw.py E501 E126 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 @@ -68,7 +68,7 @@ flake8-ignore = scrapy/downloadermiddlewares/httpcache.py E501 E126 scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 - scrapy/downloadermiddlewares/redirect.py E501 W504 + scrapy/downloadermiddlewares/redirect.py E501 scrapy/downloadermiddlewares/retry.py E501 E126 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 @@ -79,7 +79,7 @@ flake8-ignore = scrapy/extensions/httpcache.py E128 E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 - scrapy/extensions/telnet.py E501 W504 + scrapy/extensions/telnet.py E501 scrapy/extensions/throttle.py E501 # scrapy/http scrapy/http/common.py E501 @@ -90,7 +90,7 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E501 E402 W504 + scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader scrapy/loader/__init__.py E501 E128 @@ -110,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W504 + scrapy/spidermiddlewares/referer.py E501 E129 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -125,7 +125,7 @@ flake8-ignore = scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 - scrapy/utils/gz.py E501 W504 + scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 @@ -234,7 +234,7 @@ flake8-ignore = tests/test_utils_datatypes.py E402 E501 tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 W504 + tests/test_utils_http.py E501 E128 tests/test_utils_iterators.py E501 E128 E129 tests/test_utils_log.py E741 tests/test_utils_python.py E501 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 41d4f25b2..5af3831a2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -17,10 +17,10 @@ class ContractsManager: self.contracts[contract.name] = contract def tested_methods_from_spidercls(self, spidercls): + is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): - if (callable(value) and value.__doc__ and - re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + if callable(value) and value.__doc__ and is_method(value.__doc__): methods.append(key) return methods diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 09ee8377e..b32afb8e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + def process_response(self, request, response, spider): - if (request.meta.get('dont_redirect', False) or - response.status in getattr(spider, 'handle_httpstatus_list', []) or - response.status in request.meta.get('handle_httpstatus_list', []) or - request.meta.get('handle_httpstatus_all', False)): + if ( + request.meta.get('dont_redirect', False) + or response.status in getattr(spider, 'handle_httpstatus_list', []) + or response.status in request.meta.get('handle_httpstatus_list', []) + or request.meta.get('handle_httpstatus_all', False) + ): return response allowed_status = (301, 302, 303, 307, 308) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 04ffd7235..1663604e7 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username.encode('utf8') and - credentials.checkPassword(self.password.encode('utf8'))): + if not ( + credentials.username == self.username.encode('utf8') + and credentials.checkPassword(self.password.encode('utf8')) + ): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d0b5066b6..ae019c70f 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -61,8 +61,7 @@ class FilteringLinkExtractor: def __new__(cls, *args, **kwargs): from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if (issubclass(cls, FilteringLinkExtractor) and - not issubclass(cls, LxmlLinkExtractor)): + if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 3784de885..434067b00 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): - if ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + if ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) @@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + elif ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c291ae237..fbd7bd18f 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -52,8 +52,7 @@ def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return (_is_gzipped(ctype) or - (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') def gzip_magic_number(response): diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2fac3da1f..363b015a8 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual( + body, + "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" + )