From a70ec30e19eff13d3bf18e5c392612db018d1b4f Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Wed, 1 Mar 2017 23:02:42 +0100 Subject: [PATCH 001/179] Adding new options. --- scrapy/pipelines/files.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..7041a0a78 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -40,7 +40,6 @@ class FileException(Exception): class FSFilesStore(object): - def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -79,9 +78,12 @@ class FSFilesStore(object): class S3FilesStore(object): - AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_ENDPOINT_URL = None + AWS_REGION_NAME = None + AWS_USE_SSL = None + AWS_VERIFY = None POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in # FilesPipeline.from_settings. @@ -95,8 +97,14 @@ class S3FilesStore(object): import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY) + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection @@ -181,7 +189,7 @@ class S3FilesStore(object): 'X-Amz-Grant-Read': 'GrantRead', 'X-Amz-Grant-Read-ACP': 'GrantReadACP', 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', - }) + }) extra = {} for key, value in six.iteritems(headers): try: @@ -226,7 +234,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -256,6 +264,10 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] store_uri = settings['FILES_STORE'] @@ -423,4 +435,5 @@ class FilesPipeline(MediaPipeline): # deprecated def file_key(self, url): return self.file_path(url) + file_key._base = True From 605691792f5198d75524a1ad952489193970d4a7 Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:35:39 +0100 Subject: [PATCH 002/179] Updating media-pipeline docs for S3-like storage. --- docs/topics/media-pipeline.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..733a7fe2b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -169,6 +169,18 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. +Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like +self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: + + AWS_ENDPOINT_URL = 'http://minio.example.com:9000' + +For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:: + + AWS_USE_SSL = False # or True (None by default) + AWS_VERIFY = False # or True (None by default) + +.. _Minio: https://github.com/minio/minio +.. _s3.scality: https://s3.scality.com/ .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Usage example From 11cf6ad4258250b8a2410029eb794ed892dc4cea Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:48:06 +0100 Subject: [PATCH 003/179] Comments for AWS_ENDPOINT_URL setting. --- docs/topics/settings.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ccdd02c4e..3ac3fd5af 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend `. .. setting:: BOT_NAME +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. + +.. setting:: AWS_ENDPOINT_URL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. +By default SSL will be used. + +.. setting:: AWS_USE_SSL + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. + +.. setting:: AWS_VERIFY + BOT_NAME -------- From c8dc158697453b610f168f97ce94945892163674 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 30 May 2017 17:22:23 +0200 Subject: [PATCH 004/179] Use HTTP pool and proper endpoint key for ProxyAgent --- scrapy/core/downloader/handlers/http11.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9bfdd803c..a4b077b57 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed + HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -228,10 +228,33 @@ class TunnelingAgent(Agent): headers, bodyProducer, requestPath) +class ScrapyProxyAgent(Agent): + + def __init__(self, reactor, proxyURI, + connectTimeout=None, bindAddress=None, pool=None): + super(ScrapyProxyAgent, self).__init__(reactor, + connectTimeout=connectTimeout, + bindAddress=bindAddress, + pool=pool) + self._proxyURI = URI.fromBytes(proxyURI) + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + proxyEndpoint = self._getEndpoint(self._proxyURI) + key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) + return self._requestWithEndpoint(key, proxyEndpoint, method, + URI.fromBytes(uri), headers, + bodyProducer, uri) + + class ScrapyAgent(object): _Agent = Agent - _ProxyAgent = ProxyAgent + _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, @@ -260,9 +283,8 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, - timeout=timeout, bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + return self._ProxyAgent(reactor, proxyURI=proxy, + connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) From e162c1ff40bf74802c1a9f7b93fdbb56a5ff4a13 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 11:58:17 +0200 Subject: [PATCH 005/179] Pass proxy URI to ProxyAgent as bytes --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a4b077b57..7b77d82da 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,7 +283,7 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - return self._ProxyAgent(reactor, proxyURI=proxy, + return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, From 6b092c66809ec11245d03063961ca64e488580e1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 12:29:01 +0200 Subject: [PATCH 006/179] Handle Twisted versions before 15.0 --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7b77d82da..216671f82 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed, URI + HTTPConnectionPool, ResponseFailed +try: + from twisted.web.client import URI +except ImportError: + from twisted.web.client import _URI as URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -244,7 +248,12 @@ class ScrapyProxyAgent(Agent): """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - proxyEndpoint = self._getEndpoint(self._proxyURI) + if twisted_version >= (15, 0, 0): + proxyEndpoint = self._getEndpoint(self._proxyURI) + else: + proxyEndpoint = self._getEndpoint(self._proxyURI.scheme, + self._proxyURI.host, + self._proxyURI.port) key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) return self._requestWithEndpoint(key, proxyEndpoint, method, URI.fromBytes(uri), headers, From 33dfac50185b1940dda89bec3e3480a7e76e9ca7 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 24 Jul 2017 22:06:17 +0200 Subject: [PATCH 007/179] xrange() --> range() for Python 3 Either this PR or #2845. --- extras/qpsclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index bb83588dd..7554f7eec 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -41,7 +41,7 @@ class QPSSpider(Spider): slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)] + urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] else: urls = [url] From 11a1f970b7b7b68b3d968df4b29c4269ab220ac6 Mon Sep 17 00:00:00 2001 From: Pengyu Chen Date: Wed, 26 Jul 2017 16:11:13 +0800 Subject: [PATCH 008/179] Added: HTTP status code 522/524 to retry. --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 87dbf6974..697314b7f 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False From 5dc9a88c347db3497b03949938184ca339f4e9cb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:10:58 +0200 Subject: [PATCH 009/179] Handle HTTP 308 Permanent Redirect --- scrapy/downloadermiddlewares/redirect.py | 4 ++-- tests/test_downloadermiddleware_redirect.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..30cae3fee 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) + allowed_status = (301, 302, 303, 307, 308) if 'Location' not in response.headers or response.status not in allowed_status: return response @@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) - if response.status in (301, 307) or request.method == 'HEAD': + if response.status in (301, 307, 308) or request.method == 'HEAD': redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e8c92affa..a2da4aa8f 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase): req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority - def test_redirect_301(self): - def _test(method): - url = 'http://www.example.com/301' + def test_redirect_3xx_permanent(self): + def _test(method, status=301): + url = 'http://www.example.com/{}'.format(status) url2 = 'http://www.example.com/redirected' req = Request(url, method=method) - rsp = Response(url, headers={'Location': url2}, status=301) + rsp = Response(url, headers={'Location': url2}, status=status) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=308) + _test('POST', status=308) + _test('HEAD', status=308) + def test_dont_redirect(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' From 1fdc10684fc427e5446350cada465ec934330a3f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:25:11 +0200 Subject: [PATCH 010/179] HTTP Cache: treat 308 as 301 --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 12b69860a..22946b98c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest): (True, 203, {'Last-Modified': self.yesterday}), (True, 300, {'Last-Modified': self.yesterday}), (True, 301, {'Last-Modified': self.yesterday}), + (True, 308, {'Last-Modified': self.yesterday}), (True, 401, {'Last-Modified': self.yesterday}), (True, 404, {'Cache-Control': 'public, max-age=600'}), (True, 302, {'Expires': self.tomorrow}), From 15a5c533fa6f448b7e5cd72ef099725c2295ef6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Jul 2017 19:07:57 +0200 Subject: [PATCH 011/179] Add tests for HTTP 307 permanent redirects --- tests/test_downloadermiddleware_redirect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index a2da4aa8f..35e474418 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=307) + _test('POST', status=307) + _test('HEAD', status=307) + _test('GET', status=308) _test('POST', status=308) _test('HEAD', status=308) From 219c8aa0b622260b9814379be41132367ef33e39 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 17:30:30 +0200 Subject: [PATCH 012/179] Log versions information at startup --- scrapy/commands/version.py | 40 +++---------------------------- scrapy/utils/log.py | 48 +++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index e22f98f5a..92b70d88b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,12 +1,8 @@ from __future__ import print_function -import sys -import platform - -import twisted -import OpenSSL import scrapy from scrapy.commands import ScrapyCommand +from scrapy.utils.log import scrapy_components_versions class Command(ScrapyCommand): @@ -27,38 +23,8 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - import cssselect - import parsel - import lxml.etree - import w3lib - - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - try: - w3lib_version = w3lib.__version__ - except AttributeError: - w3lib_version = "<1.14.3" - - print("Scrapy : %s" % scrapy.__version__) - print("lxml : %s" % lxml_version) - print("libxml2 : %s" % libxml2_version) - print("cssselect : %s" % cssselect.__version__) - print("parsel : %s" % parsel.__version__) - print("w3lib : %s" % w3lib_version) - print("Twisted : %s" % twisted.version.short()) - print("Python : %s" % sys.version.replace("\n", "- ")) - print("pyOpenSSL : %s" % self._get_openssl_version()) - print("Platform : %s" % platform.platform()) + for name, version in scrapy_components_versions(): + print("%-9s : %s" % (name, version)) else: print("Scrapy %s" % scrapy.__version__) - def _get_openssl_version(self): - try: - openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ - .decode('ascii', errors='replace') - # pyOpenSSL 0.12 does not expose openssl version - except AttributeError: - openssl = 'Unknown OpenSSL version' - - return '{} ({})'.format(OpenSSL.version.__version__, openssl) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6ceb61a82..660b3c9f5 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -139,10 +139,56 @@ def _get_handler(settings): return handler +def scrapy_components_versions(): + import platform + + import cssselect + import parsel + import lxml.etree + import twisted + import w3lib + + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + + return [ + ("Scrapy", scrapy.__version__), + ("lxml", lxml_version), + ("libxml2", libxml2_version), + ("cssselect", cssselect.__version__), + ("parsel", parsel.__version__), + ("w3lib", w3lib_version), + ("Twisted", twisted.version.short()), + ("Python", sys.version.replace("\n", "- ")), + ("pyOpenSSL", _get_openssl_version()), + ("Platform", platform.platform()), + ] + + +def _get_openssl_version(): + try: + import OpenSSL + openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ + .decode('ascii', errors='replace') + # pyOpenSSL 0.12 does not expose openssl version + except AttributeError: + openssl = 'Unknown OpenSSL version' + + return '{} ({})'.format(OpenSSL.version.__version__, openssl) + + def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - + logger.info("Versions: %(versions)s}", + {'versions': ", ".join("%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy")}) d = dict(overridden_settings(settings)) logger.info("Overridden settings: %(settings)r", {'settings': d}) From bf7ef3e4c3ea70a60f6e5f9e5c8a6d842e2b72b9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 20:07:14 +0200 Subject: [PATCH 013/179] Move methods to a new scrapy.utils.versions --- scrapy/commands/version.py | 2 +- scrapy/utils/log.py | 45 ++------------------------------------ scrapy/utils/versions.py | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 44 deletions(-) create mode 100644 scrapy/utils/versions.py diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 92b70d88b..71b1026fa 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -2,7 +2,7 @@ from __future__ import print_function import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.log import scrapy_components_versions +from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 660b3c9f5..905c1bfc1 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -11,6 +11,8 @@ from twisted.python import log as twisted_log import scrapy from scrapy.settings import overridden_settings, Settings from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.versions import scrapy_components_versions + logger = logging.getLogger(__name__) @@ -139,49 +141,6 @@ def _get_handler(settings): return handler -def scrapy_components_versions(): - import platform - - import cssselect - import parsel - import lxml.etree - import twisted - import w3lib - - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - try: - w3lib_version = w3lib.__version__ - except AttributeError: - w3lib_version = "<1.14.3" - - return [ - ("Scrapy", scrapy.__version__), - ("lxml", lxml_version), - ("libxml2", libxml2_version), - ("cssselect", cssselect.__version__), - ("parsel", parsel.__version__), - ("w3lib", w3lib_version), - ("Twisted", twisted.version.short()), - ("Python", sys.version.replace("\n", "- ")), - ("pyOpenSSL", _get_openssl_version()), - ("Platform", platform.platform()), - ] - - -def _get_openssl_version(): - try: - import OpenSSL - openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ - .decode('ascii', errors='replace') - # pyOpenSSL 0.12 does not expose openssl version - except AttributeError: - openssl = 'Unknown OpenSSL version' - - return '{} ({})'.format(OpenSSL.version.__version__, openssl) - - def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py new file mode 100644 index 000000000..d2cff09fe --- /dev/null +++ b/scrapy/utils/versions.py @@ -0,0 +1,44 @@ +import platform +import sys + +import cssselect +import lxml.etree +import parsel +import twisted +import w3lib + +import scrapy + + +def scrapy_components_versions(): + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + + return [ + ("Scrapy", scrapy.__version__), + ("lxml", lxml_version), + ("libxml2", libxml2_version), + ("cssselect", cssselect.__version__), + ("parsel", parsel.__version__), + ("w3lib", w3lib_version), + ("Twisted", twisted.version.short()), + ("Python", sys.version.replace("\n", "- ")), + ("pyOpenSSL", _get_openssl_version()), + ("Platform", platform.platform()), + ] + + +def _get_openssl_version(): + try: + import OpenSSL + openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ + .decode('ascii', errors='replace') + # pyOpenSSL 0.12 does not expose openssl version + except AttributeError: + openssl = 'Unknown OpenSSL version' + + return '{} ({})'.format(OpenSSL.version.__version__, openssl) From aaaa4da7a4e1fe75396028189de88fb9a6604200 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 May 2017 12:42:54 +0200 Subject: [PATCH 014/179] Add template for a downloader middleware --- .../project/module/middlewares.py.tmpl | 47 +++++++++++++++++++ .../templates/project/module/settings.py.tmpl | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 292bf572e..1a4b0caa5 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -54,3 +54,50 @@ class ${ProjectName}SpiderMiddleware(object): def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) + + +class ${ProjectName}DownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 486df6b71..35a0f9a45 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -53,7 +53,7 @@ ROBOTSTXT_OBEY = True # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, +# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions From a65fec050ae4a07d233ae886457daf10f14929fe Mon Sep 17 00:00:00 2001 From: simik-ru Date: Sun, 30 Jul 2017 17:04:02 +0300 Subject: [PATCH 015/179] Small fix in description of startproject arguments --- docs/topics/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8de858f8a..dc8067d7e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -187,7 +187,7 @@ startproject Creates a new Scrapy project named ``project_name``, under the ``project_dir`` directory. -If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``. +If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``. Usage example:: From 6e6b5cc29f15dbf4f1941fca70dd9c126e4ba556 Mon Sep 17 00:00:00 2001 From: Andrei Petre Date: Tue, 1 Aug 2017 17:14:43 +0300 Subject: [PATCH 016/179] Use getfullargspec under the scenes for py3 to stop DeprecationWarning (#2864) Use getfullargspec under the scenes for py3 to stop DeprecationWarning. Closes #2862 --- scrapy/utils/python.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 72f8f4311..732ca13a0 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,6 @@ import errno import six from functools import partial, wraps import sys -import time from scrapy.utils.decorators import deprecated @@ -198,10 +197,30 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) +def _getargspec_py23(func): + """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, + defaults) + + Identical to inspect.getargspec() in python2, but uses + inspect.getfullargspec() for python3 behind the scenes to avoid + DeprecationWarning. + + >>> def f(a, b=2, *ar, **kw): + ... pass + + >>> _getargspec_py23(f) + ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) + """ + if six.PY2: + return inspect.getargspec(func) + + return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) + + def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = inspect.getargspec(func) + func_args, _, _, _ = _getargspec_py23(func) elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): @@ -248,9 +267,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = inspect.getargspec(func) + spec = _getargspec_py23(func) elif hasattr(func, '__call__'): - spec = inspect.getargspec(func.__call__) + spec = _getargspec_py23(func.__call__) else: raise TypeError('%s is not callable' % type(func)) From 71d5b7d75a579360ab02b9f9594199993ff91f61 Mon Sep 17 00:00:00 2001 From: david watson Date: Tue, 1 Aug 2017 13:49:22 -0400 Subject: [PATCH 017/179] fix typo (#2867) --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 040c8cfde..eb02086dc 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -20,7 +20,7 @@ These are some common properties often found in broad crawls: * they crawl many domains (often, unbounded) instead of a specific set of sites -* they don't necessarily crawl domains to completion, because it would +* they don't necessarily crawl domains to completion, because it would be impractical (or impossible) to do so, and instead limit the crawl by time or number of pages crawled From 01ac8838934071b89c5c711688f3d833387e27d4 Mon Sep 17 00:00:00 2001 From: Eugene Vorobev Date: Wed, 26 Jul 2017 19:51:27 +0300 Subject: [PATCH 018/179] Follow alternate link for all types of sitemaps --- scrapy/spiders/sitemap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index e54001d88..0ee8ba5e7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -48,7 +48,7 @@ class SitemapSpider(Spider): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s): + for loc in iterloc(s, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) From 0cb3085f8453265a8f37de684e62eda8b5398c75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 3 Aug 2017 16:35:47 +0200 Subject: [PATCH 019/179] Add test for alternate links --- tests/test_spider.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..2507964b5 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -348,6 +348,33 @@ Sitemap: /sitemap-relative-url.xml 'http://example.com/sitemap-uppercase.xml', 'http://www.example.com/sitemap-relative-url.xml']) + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + + spider.sitemap_alternate_links = True + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/italiano/']) + class DeprecationTest(unittest.TestCase): From c016a4309dbb045c17842e329043b7d9951e8f14 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 4 Aug 2017 01:44:23 +0200 Subject: [PATCH 020/179] # noqa to close #2836 Marks #2836 as will not fix. --- tests/test_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_item.py b/tests/test_item.py index 85a554de0..3c645649c 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ + return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 MyItem() From 0a69a32b5a2cab7575fdbb5f2cd4b7c7b900aabc Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 4 Aug 2017 14:35:43 +0200 Subject: [PATCH 021/179] Force Travis CI to test again --- tests/test_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_item.py b/tests/test_item.py index 3c645649c..2c1eb0dd3 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 + return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 MyItem() From be71f98e92688c759d3af48101617229dcdfe05f Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Sat, 29 Jul 2017 20:51:54 -0400 Subject: [PATCH 022/179] Explicit message for scrapy parse callback The scrapy parse method raises a NotImplementedError when not defined, but for new comers it can be hard to debug what might be going wrong. Adding an explicit message for NotImplementedError will help new users. --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30cb7590a..c6b92f8eb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError + raise NotImplementedError('Spider.parse callback is not defined') @classmethod def update_settings(cls, settings): From 7adab61a7a5f88c78311cc44a468c7b8d0a4c954 Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Tue, 1 Aug 2017 22:42:26 -0400 Subject: [PATCH 023/179] Added test for NotImplemented Spider.parse method --- tests/test_spider.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..6c845d826 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -429,3 +429,17 @@ class DeprecationTest(unittest.TestCase): self.assertEqual(len(requests), 1) self.assertEqual(requests[0].url, 'http://example.com/foo') self.assertEqual(len(w), 1) + + +class NoParseMethodSpiderTest(unittest.TestCase): + + spider_class = Spider + + def test_undefined_parse_method(self): + spider = self.spider_class('example.com') + text = 'Random text response' + resp = TextResponse(url="http://www.example.com/random_url", body=text) + + exc_msg = 'Spider.parse callback is not defined' + with self.assertRaisesRegexp(NotImplementedError, exc_msg): + spider.parse(resp) From 12409a0cf6c37ff5c19588bb064690549798bb37 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Wed, 2 Aug 2017 08:32:38 -0400 Subject: [PATCH 024/179] Fix broken encoding on text for py 3 --- tests/test_spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 6c845d826..6a52b3ea7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -437,7 +437,7 @@ class NoParseMethodSpiderTest(unittest.TestCase): def test_undefined_parse_method(self): spider = self.spider_class('example.com') - text = 'Random text response' + text = b'Random text' resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = 'Spider.parse callback is not defined' From 2960c9b5683dceb149823e7f927d8c86ee83deb8 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Sat, 5 Aug 2017 16:29:41 -0400 Subject: [PATCH 025/179] Use self.__class__.__name__ instead of showing generic Spider class name --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c6b92f8eb..e9c131e3b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError('Spider.parse callback is not defined') + raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod def update_settings(cls, settings): From 4ca61a20512b7306e7266e6667a0c02ae5ebe557 Mon Sep 17 00:00:00 2001 From: Chomba Ng'ang'a Date: Mon, 7 Aug 2017 18:29:36 +0300 Subject: [PATCH 026/179] Update deprecated test aliases - change ``failIf`` to ``assertFalse`` - change ``asertEquals`` to ``assertEqual`` - change ``assert_`` to ``assertTrue`` https://docs.python.org/2/library/unittest.html#deprecated-aliases --- tests/test_cmdline/__init__.py | 2 +- tests/test_downloader_handlers.py | 88 +++++++++---------- tests/test_downloadermiddleware_cookies.py | 20 ++--- ...est_downloadermiddleware_defaultheaders.py | 6 +- ...st_downloadermiddleware_downloadtimeout.py | 8 +- tests/test_downloadermiddleware_httpauth.py | 4 +- ...st_downloadermiddleware_httpcompression.py | 2 +- tests/test_downloadermiddleware_httpproxy.py | 44 +++++----- tests/test_downloadermiddleware_redirect.py | 4 +- tests/test_downloadermiddleware_useragent.py | 6 +- tests/test_http_cookies.py | 2 +- tests/test_http_request.py | 4 +- tests/test_loader.py | 18 ++-- tests/test_pipeline_images.py | 16 ++-- tests/test_selector.py | 2 +- tests/test_spider.py | 6 +- tests/test_spidermiddleware_depth.py | 8 +- tests/test_spidermiddleware_httperror.py | 24 ++--- tests/test_spidermiddleware_offsite.py | 6 +- tests/test_spidermiddleware_referer.py | 10 +-- tests/test_spidermiddleware_urllength.py | 2 +- tests/test_urlparse_monkeypatches.py | 8 +- tests/test_utils_datatypes.py | 8 +- tests/test_utils_defer.py | 2 +- tests/test_utils_iterators.py | 4 +- tests/test_utils_misc/__init__.py | 8 +- tests/test_utils_project.py | 8 +- tests/test_utils_python.py | 26 +++--- tests/test_utils_signal.py | 2 +- tests/test_webclient.py | 28 +++--- 30 files changed, 188 insertions(+), 188 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 7733e7180..10076bbca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertEquals(200, settingsdict[EXT_PATH]) + self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7d88dbcba..bd2c86292 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -98,9 +98,9 @@ class FileTestCase(unittest.TestCase): def test_download(self): def _test(response): - self.assertEquals(response.url, request.url) - self.assertEquals(response.status, 200) - self.assertEquals(response.body, b'0123456789') + self.assertEqual(response.url, request.url) + self.assertEqual(response.status, 200) + self.assertEqual(response.body, b'0123456789') request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -241,28 +241,28 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'') + d.addCallback(self.assertEqual, b'') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d @defer.inlineCallbacks @@ -285,24 +285,24 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals( + self.assertEqual( response.body, to_bytes('%s:%d' % (self.host, self.portno))) - self.assertEquals(request.headers, {}) + self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): - self.assertEquals(response.body, b'example.com') - self.assertEquals(request.headers.get('Host'), b'example.com') + self.assertEqual(response.body, b'example.com') + self.assertEqual(request.headers.get('Host'), b'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'example.com') + d.addCallback(self.assertEqual, b'example.com') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -317,7 +317,7 @@ class HttpTestCase(unittest.TestCase): https://bugs.python.org/issue14721 """ def _test(response): - self.assertEquals(response.body, b'0') + self.assertEqual(response.body, b'0') request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -327,8 +327,8 @@ class HttpTestCase(unittest.TestCase): import json headers = Headers(json.loads(response.text)['headers']) contentlengths = headers.getlist('Content-Length') - self.assertEquals(len(contentlengths), 1) - self.assertEquals(contentlengths, [b"0"]) + self.assertEqual(len(contentlengths), 1) + self.assertEqual(contentlengths, [b"0"]) request = Request(self.getURL('echo'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) @@ -338,7 +338,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, body) + d.addCallback(self.assertEqual, body) return d @@ -364,7 +364,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_response_class_choosing_request(self): @@ -374,7 +374,7 @@ class Http11TestCase(HttpTestCase): body = b'Some plain text\ndata with tabs\t and null bytes\0' def _test_type(response): - self.assertEquals(type(response), TextResponse) + self.assertEqual(type(response), TextResponse) request = Request(self.getURL('nocontenttype'), body=body) d = self.download_request(request, Spider('foo')) @@ -389,7 +389,7 @@ class Http11TestCase(HttpTestCase): # response body. (regardless of headers) d = self.download_request(request, Spider('foo', download_maxsize=10)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") yield d d = self.download_request(request, Spider('foo', download_maxsize=9)) @@ -431,14 +431,14 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo', download_maxsize=100)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_download_chunked_content(self): request = Request(self.getURL('chunked')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"chunked content\n") + d.addCallback(self.assertEqual, b"chunked content\n") return d def test_download_broken_content_cause_data_loss(self, url='broken'): @@ -597,9 +597,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'http://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'http://example.com') http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -607,9 +607,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy_https_noconnect(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'https://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) @@ -617,9 +617,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_without_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'/path/to/resource') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/path/to/resource') request = Request(self.getURL('path/to/resource')) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -978,7 +978,7 @@ class DataURITestCase(unittest.TestCase): uri = "data:,A%20brief%20note" def _test(response): - self.assertEquals(response.url, uri) + self.assertEqual(response.url, uri) self.assertFalse(response.headers) request = Request(uri) @@ -986,39 +986,39 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): - self.assertEquals(response.text, 'A brief note') - self.assertEquals(type(response), + self.assertEqual(response.text, 'A brief note') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "US-ASCII") + self.assertEqual(response.encoding, "US-ASCII") request = Request("data:,A%20brief%20note") return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(type(response), + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "iso-8859-7") + self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_text_charset(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(response.body, b'\xbe\xd3\xbe') - self.assertEquals(response.encoding, "iso-8859-7") + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.body, b'\xbe\xd3\xbe') + self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_mediatype_parameters(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(type(response), + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "utf-8") + self.assertEqual(response.encoding, "utf-8") request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' @@ -1027,7 +1027,7 @@ class DataURITestCase(unittest.TestCase): def test_base64(self): def _test(response): - self.assertEquals(response.text, 'Hello, world.') + self.assertEqual(response.text, 'Hello, world.') request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 26d9794b6..17801e502 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -36,7 +36,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') assert self.mw.process_request(req2, self.spider) is None - self.assertEquals(req2.headers.get('Cookie'), b"C1=value1") + self.assertEqual(req2.headers.get('Cookie'), b"C1=value1") def test_setting_false_cookies_enabled(self): self.assertRaises( @@ -131,12 +131,12 @@ class CookiesMiddlewareTest(TestCase): # check that cookies are merged back req = Request('http://scrapytest.org/mergeme') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') def test_complex_cookies(self): # merge some cookies into jar @@ -157,7 +157,7 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request('http://scrapytest.org/bar') self.mw.process_request(req, self.spider) - self.assertEquals(req.headers.get('Cookie'), b'C2=value2') + self.assertEqual(req.headers.get('Cookie'), b'C2=value2') # embed nothing for scrapytest.org/baz req = Request('http://scrapytest.org/baz') @@ -167,7 +167,7 @@ class CookiesMiddlewareTest(TestCase): def test_merge_request_cookies(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers) @@ -181,7 +181,7 @@ class CookiesMiddlewareTest(TestCase): def test_cookiejar_key(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers, request=req) @@ -193,7 +193,7 @@ class CookiesMiddlewareTest(TestCase): req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) assert self.mw.process_request(req3, self.spider) is None - self.assertEquals(req3.headers.get('Cookie'), b'galleta=dulce') + self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') headers = {'Set-Cookie': 'C2=value2; path=/'} res2 = Response('http://scrapytest.org/', headers=headers, request=req3) @@ -213,16 +213,16 @@ class CookiesMiddlewareTest(TestCase): req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') assert self.mw.process_request(req5_2, self.spider) is None - self.assertEquals(req5_2.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1') req5_3 = Request('http://scrapytest.org/some-redirected-path') assert self.mw.process_request(req5_3, self.spider) is None - self.assertEquals(req5_3.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1') #skip cookie retrieval for not http request req6 = Request('file:///scrapy/sometempfile') assert self.mw.process_request(req6, self.spider) is None - self.assertEquals(req6.headers.get('Cookie'), None) + self.assertEqual(req6.headers.get('Cookie'), None) def test_local_domain(self): request = Request("http://example-host/", cookies={'currencyCookie': 'USD'}) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 80efa83f9..6a31dfcf8 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -22,15 +22,15 @@ class TestDefaultHeadersMiddleware(TestCase): defaults, spider, mw = self.get_defaults_spider_mw() req = Request('http://www.scrapytest.org') mw.process_request(req, spider) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']} req = Request('http://www.scrapytest.org', headers=headers) - self.assertEquals(req.headers, bytes_headers) + self.assertEqual(req.headers, bytes_headers) mw.process_request(req, spider) defaults.update(bytes_headers) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 446a99f36..586bdc0d1 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -18,20 +18,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 180) + self.assertEqual(req.meta.get('download_timeout'), 180) def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 20.1) + self.assertEqual(req.meta.get('download_timeout'), 20.1) def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 2) + self.assertEqual(req.meta.get('download_timeout'), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -39,4 +39,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): mw.spider_opened(spider) req.meta['download_timeout'] = 1 assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 1) + self.assertEqual(req.meta.get('download_timeout'), 1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 425a5cc79..3381632b0 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -23,10 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') def test_auth_already_set(self): req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0678fcb14..0745c8dd3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -248,4 +248,4 @@ class HttpCompressionTest(TestCase): response = response.replace(body = None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) - self.assertEquals(response.body, b'') + self.assertEqual(response.body, b'') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index c77179ceb..0ea83aaf9 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -35,8 +35,8 @@ class TestDefaultHeadersMiddleware(TestCase): for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta, {}) + self.assertEqual(req.url, url) + self.assertEqual(req.meta, {}) def test_enviroment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' @@ -48,41 +48,41 @@ class TestDefaultHeadersMiddleware(TestCase): ('https://e.com', https_proxy), ('file://tmp/a', None)]: req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta.get('proxy'), proxy) + self.assertEqual(req.url, url) + self.assertEqual(req.meta.get('proxy'), proxy) def test_proxy_precedence_meta(self): os.environ['http_proxy'] = 'https://proxy.com' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'}) def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -90,27 +90,27 @@ class TestDefaultHeadersMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') def test_proxy_already_seted(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' @@ -142,4 +142,4 @@ class TestDefaultHeadersMiddleware(TestCase): os.environ['no_proxy'] = '*' req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) + self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 35e474418..74137b4cd 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -166,7 +166,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) def test_utf8_location(self): req = Request('http://scrapytest.org/first') @@ -174,7 +174,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) class MetaRefreshMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 1e41fdace..a286764fd 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw('default_useragent') req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'default_useragent') + self.assertEqual(req.headers['User-Agent'], b'default_useragent') def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent @@ -34,7 +34,7 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'spider_useragent') + self.assertEqual(req.headers['User-Agent'], b'spider_useragent') def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') @@ -43,7 +43,7 @@ class UserAgentMiddlewareTest(TestCase): req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'header_useragent') + self.assertEqual(req.headers['User-Agent'], b'header_useragent') def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 549f779d8..caa6fe83e 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase): self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assert_(self.wrapped.info() is self.wrapped) + self.assertTrue(self.wrapped.info() is self.wrapped) def test_getheaders(self): self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9b0ee63dc..21c0dd746 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase): h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' for k, v in h.iteritems(): - self.assert_(isinstance(k, bytes)) + self.assertTrue(isinstance(k, bytes)) for s in v: - self.assert_(isinstance(s, bytes)) + self.assertTrue(isinstance(s, bytes)) def test_eq(self): url = 'http://www.scrapy.org' diff --git a/tests/test_loader.py b/tests/test_loader.py index 9d07eb95b..2569ccf5e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assert_(isinstance(proc(['hello', 'world']), six.text_type)) + self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type)) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertTrue(l.selector is sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -490,21 +490,21 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertTrue(l.selector is sel) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -526,7 +526,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//p/text()') @@ -552,7 +552,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//div/text()', re='ma') @@ -568,7 +568,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_css('name', 'p::text') @@ -606,7 +606,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0f3047602..03c6d8059 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -81,28 +81,28 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255) im = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEquals(thumbnail.mode, 'RGB') - self.assertEquals(thumbnail.size, (10, 10)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) im = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) class DeprecatedImagesPipeline(ImagesPipeline): diff --git a/tests/test_selector.py b/tests/test_selector.py index af0cc4de2..526660cc8 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -84,7 +84,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_badly_encoded_body(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..0a343549e 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -207,7 +207,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) @@ -234,7 +234,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html']) @@ -258,7 +258,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index a3cdc0114..3685d5a6f 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -25,18 +25,18 @@ class TestDepthMiddleware(TestCase): result = [Request('http://scrapytest.org')] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out, result) + self.assertEqual(out, result) rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) - self.assertEquals(rdc, 1) + self.assertEqual(rdc, 1) req.meta['depth'] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out2, []) + self.assertEqual(out2, []) rdm = self.stats.get_value('request_depth_max', spider=self.spider) - self.assertEquals(rdm, 1) + self.assertEqual(rdm, 1) def tearDown(self): self.stats.close_spider(self.spider, '') diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e1407e6b3..19e6bbdcd 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -67,16 +67,16 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEquals([], + self.assertEqual([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider)) @@ -84,11 +84,11 @@ class TestHttpErrorMiddleware(TestCase): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) @@ -102,11 +102,11 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -117,14 +117,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) @@ -139,9 +139,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): @@ -152,7 +152,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 37c3a450b..9ad86313c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -37,7 +37,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, onsite_reqs) + self.assertEqual(out, onsite_reqs) class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -49,7 +49,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response('http://scrapytest.org') reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -67,4 +67,4 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response('http://scrapytest.org') reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f27f31b74..21439c20e 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -45,7 +45,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + self.assertEqual(out[0].headers.get('Referer'), referrer) class MixinDefault(object): @@ -490,7 +490,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) - self.assertEquals(mw.default_policy, p) + self.assertEqual(mw.default_policy, p) def test_valid_name_casevariants(self): for s, p in [ @@ -506,7 +506,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) - self.assertEquals(mw.default_policy, p) + self.assertEqual(mw.default_policy, p) def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) @@ -581,7 +581,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): request = self.get_request(target) out = list(self.referrermw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), init_referrer) + self.assertEqual(out[0].headers.get('Referer'), init_referrer) for status, url in redirections: response = Response(request.url, headers={'Location': url}, status=status) @@ -589,7 +589,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEquals(request.headers.get('Referer'), final_referrer) + self.assertEqual(request.headers.get('Referer'), final_referrer) class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index dca868ecf..a0aae0fdd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,5 +17,5 @@ class TestUrlLengthMiddleware(TestCase): mw = UrlLengthMiddleware(maxlength=25) spider = Spider('foo') out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEquals(out, [short_url_req]) + self.assertEqual(out, [short_url_req]) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 052dde37f..22e39821c 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -6,7 +6,7 @@ class UrlparseTestCase(unittest.TestCase): def test_s3_url(self): p = urlparse('s3://bucket/key/name?param=value') - self.assertEquals(p.scheme, 's3') - self.assertEquals(p.hostname, 'bucket') - self.assertEquals(p.path, '/key/name') - self.assertEquals(p.query, 'param=value') + self.assertEqual(p.scheme, 's3') + self.assertEqual(p.hostname, 'bucket') + self.assertEqual(p.path, '/key/name') + self.assertEqual(p.query, 'param=value') diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 49323f0ff..5b83869b8 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -202,22 +202,22 @@ class SequenceExcludeTest(unittest.TestCase): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEquals([], are_not_in) + self.assertEqual([], are_not_in) are_not_in = [v for v in range(10, 20) if v in d] - self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index f49bbfafe..003bb9b02 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -89,7 +89,7 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(itergood(), errors.append)) self.assertEqual(out, list(range(10))) - self.failIf(errors) + self.assertFalse(errors) def test_iter_errback_bad(self): def iterbad(): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e3889a4..b2e8610f8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -252,8 +252,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..832253aa4 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -23,20 +23,20 @@ class UtilsMiscTestCase(unittest.TestCase): 'tests.test_utils_misc.test_walk_modules.mod.mod0', 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') expected = [ 'tests.test_utils_misc.test_walk_modules.mod', 'tests.test_utils_misc.test_walk_modules.mod.mod0', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') expected = [ 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) self.assertRaises(ImportError, walk_modules, 'nomodule999') @@ -51,7 +51,7 @@ class UtilsMiscTestCase(unittest.TestCase): 'testegg.spiders.b', 'testegg' ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) finally: sys.path.remove(egg) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 6b7fcd4c2..7e2caace8 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,14 +25,14 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEquals('.scrapy/somepath', data_path('somepath')) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('.scrapy/somepath', data_path('somepath')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) def test_data_path_inside_project(self): with inside_a_project() as proj_path: expected = os.path.join(proj_path, '.scrapy', 'somepath') - self.assertEquals( + self.assertEqual( os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 8becca0f1..c2e4037e8 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -97,9 +97,9 @@ class UtilsPythonTestCase(unittest.TestCase): a = Obj() b = Obj() # no attributes given return False - self.failIf(equal_attributes(a, b, [])) + self.assertFalse(equal_attributes(a, b, [])) # not existent attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.x = 1 b.x = 1 @@ -108,7 +108,7 @@ class UtilsPythonTestCase(unittest.TestCase): b.y = 2 # obj1 has no attribute y - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.y = 2 # equal attributes @@ -116,7 +116,7 @@ class UtilsPythonTestCase(unittest.TestCase): a.y = 1 # differente attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) # test callable a.meta = {} @@ -134,7 +134,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, [compare_z, 'x'])) # fail z equality a.meta['z'] = 2 - self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): class _Weakme(object): pass @@ -156,9 +156,9 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): @@ -166,17 +166,17 @@ class UtilsPythonTestCase(unittest.TestCase): d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b7de85049..dea81adf6 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase): self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) - self.assert_(isinstance(result[0][1], Failure)) + self.assertTrue(isinstance(result[0][1], Failure)) self.assertEqual(result[1], (self.ok_handler, "OK")) dispatcher.disconnect(self.error_handler, signal=test_signal) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3ad1aa70e..fedac2634 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -71,7 +71,7 @@ class ParseUrlTestCase(unittest.TestCase): for url, test in tests: test = tuple( to_bytes(x) if not isinstance(x, int) else x for x in test) - self.assertEquals(client._parse(url), test, url) + self.assertEqual(client._parse(url), test, url) def test_externalUnicodeInterference(self): """ @@ -258,16 +258,16 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEquals, to_bytes(s)) + self.assertEqual, to_bytes(s)) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( - self.assertEquals, to_bytes("www.example.com"))]) + self.assertEqual, to_bytes("www.example.com"))]) def test_getPage(self): """ @@ -275,7 +275,7 @@ class WebClientTestCase(unittest.TestCase): the body of the response if the default method B{GET} is used. """ d = getPage(self.getURL("file")) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_getPageHead(self): @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) return d def test_timeoutTriggering(self): @@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assert_(b'404 - No Such Resource' in pageData) + self.assertTrue(b'404 - No Such Resource' in pageData) def testFactoryInfo(self): url = self.getURL('file') @@ -336,16 +336,16 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEquals(factory.status, b'200') - self.assert_(factory.version.startswith(b'HTTP/')) - self.assertEquals(factory.message, b'OK') - self.assertEquals(factory.response_headers[b'content-length'], b'10') + self.assertEqual(factory.status, b'200') + self.assertTrue(factory.version.startswith(b'HTTP/')) + self.assertEqual(factory.message, b'OK') + self.assertEqual(factory.response_headers[b'content-length'], b'10') def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEquals(pageData, + self.assertEqual(pageData, b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') @@ -360,6 +360,6 @@ class WebClientTestCase(unittest.TestCase): def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b'Content-Encoding']) - self.assertEquals(content_encoding, EncodingResource.out_encoding) - self.assertEquals( + self.assertEqual(content_encoding, EncodingResource.out_encoding) + self.assertEqual( response.body.decode(content_encoding), to_unicode(original_body)) From fd27cde24d273e30f72f53f2711515403270838a Mon Sep 17 00:00:00 2001 From: Chomba Ng'ang'a Date: Tue, 8 Aug 2017 19:08:53 +0300 Subject: [PATCH 027/179] Update asserts to use more generic ones --- tests/test_http_cookies.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_loader.py | 6 +++--- tests/test_utils_python.py | 6 +++--- tests/test_utils_signal.py | 2 +- tests/test_webclient.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index caa6fe83e..0a9ed500a 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase): self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assertTrue(self.wrapped.info() is self.wrapped) + self.assertIs(self.wrapped.info(), self.wrapped) def test_getheaders(self): self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 21c0dd746..fca8ff411 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase): h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' for k, v in h.iteritems(): - self.assertTrue(isinstance(k, bytes)) + self.assertIsInstance(k, bytes) for s in v: - self.assertTrue(isinstance(s, bytes)) + self.assertIsInstance(s, bytes) def test_eq(self): url = 'http://www.scrapy.org' diff --git a/tests/test_loader.py b/tests/test_loader.py index 2569ccf5e..3b5714058 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type)) + self.assertIsInstance(proc(['hello', 'world']), six.text_type) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assertTrue(l.selector is sel) + self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -490,7 +490,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assertTrue(l.selector is sel) + self.assertIs(l.selector, sel) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c2e4037e8..115f523e9 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -156,7 +156,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @@ -166,7 +166,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dea81adf6..62edd420d 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase): self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) - self.assertTrue(isinstance(result[0][1], Failure)) + self.assertIsInstance(result[0][1], Failure) self.assertEqual(result[1], (self.ok_handler, "OK")) dispatcher.disconnect(self.error_handler, signal=test_signal) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fedac2634..766329b57 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assertTrue(b'404 - No Such Resource' in pageData) + self.assertIn(b'404 - No Such Resource', pageData) def testFactoryInfo(self): url = self.getURL('file') From 1dcea6a9d4615afc463fa9839d002d861aed5274 Mon Sep 17 00:00:00 2001 From: kim minji Date: Wed, 16 Aug 2017 18:07:52 +0900 Subject: [PATCH 028/179] fix typo --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7b8518832..1367536ab 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -40,7 +40,7 @@ class Crawler(object): handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) if get_scrapy_root_handler() is not None: - # scrapy root handler alread installed: update it with new settings + # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope From 1968a8ec02913273a99a3137cb419e2649d69a5f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Aug 2017 15:08:10 +0200 Subject: [PATCH 029/179] Move logging of overriden settings to Crawler init --- scrapy/crawler.py | 5 ++++- scrapy/utils/log.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1367536ab..0a56ef57a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -11,7 +11,7 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader from scrapy.extension import ExtensionManager -from scrapy.settings import Settings +from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names @@ -34,6 +34,9 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + d = dict(overridden_settings(self.settings)) + logger.info("Overridden settings: %(settings)r", {'settings': d}) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 905c1bfc1..c6d1cdf46 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -9,7 +9,7 @@ from twisted.python.failure import Failure from twisted.python import log as twisted_log import scrapy -from scrapy.settings import overridden_settings, Settings +from scrapy.settings import Settings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.versions import scrapy_components_versions @@ -148,8 +148,6 @@ def log_scrapy_info(settings): {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) - d = dict(overridden_settings(settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) class StreamLogger(object): From 7a35a1ad4ad38b2f413528b0184d8709dea2a495 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Aug 2017 17:08:21 +0200 Subject: [PATCH 030/179] Remove trailing bracket from components versions log --- scrapy/utils/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 905c1bfc1..61b978c55 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -144,7 +144,7 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Versions: %(versions)s}", + logger.info("Versions: %(versions)s", {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) From a429d78019a379fff29c7aa3fff0a0f0427b6995 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 24 Aug 2017 16:03:36 -0300 Subject: [PATCH 031/179] update scrapinghub.com urls to use https --- docs/intro/install.rst | 2 +- docs/topics/deploy.rst | 6 +++--- docs/topics/logging.rst | 4 ++-- docs/topics/practices.rst | 2 +- docs/topics/ubuntu.rst | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 47af8292e..deb0118d4 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -235,7 +235,7 @@ After any of these workarounds you should be able to install Scrapy:: .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: http://brew.sh/ .. _zsh: http://www.zsh.org/ -.. _Scrapinghub: http://scrapinghub.com +.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: http://docs.continuum.io/anaconda/index .. _Miniconda: http://conda.pydata.org/docs/install/quick.html .. _conda-forge: https://conda-forge.github.io/ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index f4186ea7a..f2e11fe8f 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -51,9 +51,9 @@ just like ``scrapyd-deploy``. .. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/ +.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: http://doc.scrapinghub.com/shub.html +.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: http://scrapinghub.com/ +.. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a3281dd6b..0986929ad 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -102,7 +102,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -118,7 +118,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 63913d3c4..21aa4a0a7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -253,5 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html -.. _Crawlera: http://scrapinghub.com/crawlera +.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: http://scrapoxy.io/ diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 679bb56ff..81ce800aa 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -37,5 +37,5 @@ To use the packages: .. warning:: `python-scrapy` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. -.. _Scrapinghub: http://scrapinghub.com/ +.. _Scrapinghub: https://scrapinghub.com/ .. _GitHub repo: https://github.com/scrapy/scrapy From 9f16f040b661f18f2e61a35427199c099bfd2f90 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 11:53:59 +0200 Subject: [PATCH 032/179] ur'string' not needed in Py 2, syntax error in Py3 Convert `ur'Scrapy Documentation'`--> `u'Scrapy Documentation'`to be compatible with both Python 2 and Python 3. See #2891 --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 640dcd7cb..5780db65d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -191,7 +191,7 @@ htmlhelp_basename = 'Scrapydoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', ur'Scrapy Documentation', + ('index', 'Scrapy.tex', u'Scrapy Documentation', ur'Scrapy developers', 'manual'), ] From b7022360824cd1a8aa19fb0d65ea01700f06a208 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 11:56:09 +0200 Subject: [PATCH 033/179] ur'string' not needed in Py 2, syntax error in Py3 Convert `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`--> `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`to be compatible with both Python 2 and Python 3. See #2891 --- docs/utils/linkfix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 40316968f..6290adbe2 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -20,7 +20,7 @@ _filename = None _contents = None # A regex that matches standard linkcheck output lines -line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: From b8fabeed8652d22725959345700b9e7d00073de4 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 13:55:05 +0200 Subject: [PATCH 034/179] ur'string' not needed in Py 2, syntax error in Py3 This instance was missed in #2909 --> ur'Scrapy developers' --> u'Scrapy developers' --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 5780db65d..007dc2788 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -192,7 +192,7 @@ htmlhelp_basename = 'Scrapydoc' # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ ('index', 'Scrapy.tex', u'Scrapy Documentation', - ur'Scrapy developers', 'manual'), + u'Scrapy developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of From abaf466bb311f6416a58763ad7974825d88f4855 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 7 Sep 2017 11:37:40 +0200 Subject: [PATCH 035/179] Print cryptography package version --- scrapy/commands/version.py | 7 +++++-- scrapy/utils/versions.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 71b1026fa..577365c3b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,11 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - for name, version in scrapy_components_versions(): - print("%-9s : %s" % (name, version)) + versions = scrapy_components_versions() + width = max(len(n) for (n, _) in versions) + patt = "%-{}s : %s".format(width) + for name, version in versions: + print(patt % (name, version)) else: print("Scrapy %s" % scrapy.__version__) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index d2cff09fe..58c7aef85 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -17,6 +17,11 @@ def scrapy_components_versions(): w3lib_version = w3lib.__version__ except AttributeError: w3lib_version = "<1.14.3" + try: + import cryptography + cryptography_version = cryptography.__version__ + except ImportError: + cryptography_version = "unknown" return [ ("Scrapy", scrapy.__version__), @@ -28,6 +33,7 @@ def scrapy_components_versions(): ("Twisted", twisted.version.short()), ("Python", sys.version.replace("\n", "- ")), ("pyOpenSSL", _get_openssl_version()), + ("cryptography", cryptography_version), ("Platform", platform.platform()), ] From aab98080a06281ca3a88646990b81b92f492c517 Mon Sep 17 00:00:00 2001 From: Iulian Onofrei Date: Mon, 11 Sep 2017 00:40:55 +0300 Subject: [PATCH 036/179] Add option to disable automatic log handler install --- scrapy/crawler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0a56ef57a..a33ce9805 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -237,15 +237,18 @@ class CrawlerProcess(CrawlerRunner): The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. + :param install_root_handler: whether to install root logging handler + (default: True) + This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): + def __init__(self, settings=None, install_root_handler=True): super(CrawlerProcess, self).__init__(settings) install_shutdown_handlers(self._signal_shutdown) - configure_logging(self.settings) + configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) def _signal_shutdown(self, signum, _): From 3637b75a6702cb3fb4962477c0f2ec38e366f3e2 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 12 Sep 2017 15:54:09 -0400 Subject: [PATCH 037/179] [Doc] Update Response.body type --- docs/topics/request-response.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6ca37b7c9..92aae1ad0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -525,11 +525,11 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. It must be str, not unicode, unless you're - using a encoding-aware :ref:`Response subclass - `, such as - :class:`TextResponse`. - :type body: str + :param body: the response body. To access the decoded text as str (unicode + in Python 2) you can use ``response.text`` from an encoding-aware + :ref:`Response subclass `, + such as :class:`TextResponse`. + :type body: bytes :param flags: is a list containing the initial values for the :attr:`Response.flags` attribute. If given, the list will be shallow From d71a0634039d637dc10509ebb63fa2f4ef595ebb Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 12 Sep 2017 18:30:15 +0900 Subject: [PATCH 038/179] Support for Google Cloud Storage --- scrapy/pipelines/files.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index eae03752a..304d89bcf 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -194,6 +194,41 @@ class S3FilesStore(object): return extra +class GCSFilesStore(object): + + GCS_PROJECT_ID = None + + CACHE_CONTROL = 'max-age=172800' + + def __init__(self, uri): + from google.cloud import storage + client = storage.Client(project=self.GCS_PROJECT_ID) + bucket, prefix = uri[5:].split('/', 1) + self.bucket = client.bucket(bucket) + self.prefix = prefix + + def stat_file(self, path, info): + def _onsuccess(blob): + if blob: + checksum = blob.md5_hash + last_modified = time.mktime(blob.updated.timetuple()) + return {'checksum': checksum, 'last_modified': last_modified} + else: + return {} + + return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + + def persist_file(self, path, buf, info, meta=None, headers=None): + blob = self.bucket.blob(self.prefix + path) + blob.cache_control = self.CACHE_CONTROL + blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + return threads.deferToThread( + blob.upload_from_string, + data=buf.getvalue(), + content_type='application/octet-stream' + ) + + class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading @@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline): '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, + 'gs': GCSFilesStore, } DEFAULT_FILES_URLS_FIELD = 'file_urls' DEFAULT_FILES_RESULT_FIELD = 'files' @@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) From e5d4364b2a0e7ae205605905ac0c5ac6fd8d15db Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 16:24:04 +0900 Subject: [PATCH 039/179] Add tests for GCS Storage --- scrapy/utils/test.py | 15 +++++++++++++++ tests/test_pipeline_files.py | 28 +++++++++++++++++++++++++++- tox.ini | 3 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d2ef68912..60b931f48 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,6 +20,12 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") + +def assert_gcs_environ(): + if 'GCS_PROJECT_ID' not in os.environ: + raise SkipTest("GCS_PROJECT_ID not found") + + def skip_if_no_boto(): try: is_botocore() @@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content +def get_gcs_content_and_delete(bucket, path): + from google.cloud import storage + client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + bucket.delete_blob(path) + return content, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e3ec04b8d..c761bd606 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,12 +11,13 @@ from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete +from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore from tests import mock @@ -375,6 +376,31 @@ class TestS3FilesStore(unittest.TestCase): self.assertEqual(key.content_type, 'image/png') +class TestGCSFilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_gcs_environ() + uri = os.environ.get('GCS_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No GCS URI available for testing") + data = b"TestGCSFilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = 'full/filename' + store = GCSFilesStore(uri) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') + u = urlparse(uri) + content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + self.assertEqual(content, data) + self.assertEqual(blob.metadata, {'foo': 'bar'}) + self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) + self.assertEqual(blob.content_type, 'application/octet-stream') + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tox.ini b/tox.ini index c7e1e43c9..0608693ba 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ deps = -rrequirements.txt # Extras botocore + google-cloud-storage Pillow != 3.0.0 leveldb -rtests/requirements.txt @@ -18,6 +19,8 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + GCS_TEST_FILE_URI + GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} From ee166ec44f38da7f5b99c6c164a7c6ff02b37c16 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 17:35:46 +0900 Subject: [PATCH 040/179] Support for ImagesPipeline --- scrapy/pipelines/files.py | 8 +++++++- scrapy/pipelines/images.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 304d89bcf..7fdb8a086 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -218,6 +218,12 @@ class GCSFilesStore(object): return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + def _get_content_type(self, headers): + if headers and 'Content-Type' in headers: + return headers['Content-Type'] + else: + return 'application/octet-stream' + def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL @@ -225,7 +231,7 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type='application/octet-stream' + content_type=self._get_content_type(headers) ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bc449431f..c5fc12afe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) From 088b80d41a12a7b79e440cc4d4a3aae678d5c4af Mon Sep 17 00:00:00 2001 From: Renze Yu Date: Wed, 13 Sep 2017 23:29:22 +0800 Subject: [PATCH 041/179] minor fix typo --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3b3bd8d21..29f54bc86 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -452,7 +452,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting its contents. If you run this command twice without removing the file before the second time, you'll end up with a broken JSON file. -You can also used other formats, like `JSON Lines`_:: +You can also use other formats, like `JSON Lines`_:: scrapy crawl quotes -o quotes.jl From dcb279bd6cc85cf1743b548e44b050edba6a2ed8 Mon Sep 17 00:00:00 2001 From: djunzu Date: Sun, 17 Sep 2017 16:09:22 -0300 Subject: [PATCH 042/179] Add m4v extension to IGNORED_EXTENSIONS in LinkExtractor. modified: scrapy/linkextractors/__init__.py --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 8676c3b92..2d7115cc5 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', + 'm4a', 'm4v', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From 84111969c4250b486b15729b42f88300fc983511 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Sep 2017 13:35:48 +0200 Subject: [PATCH 043/179] Update pypy version regexp to get last release PyPy changed naming conention since 5.8 release, not it's called pypy2.7-x.x.x --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f44d1e6d..9c51fafb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ install: rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" fi # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1` + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1` "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" From 00c81a32ee58d0a59c14373471bf152c46131aec Mon Sep 17 00:00:00 2001 From: Craig Rodrigues Date: Sat, 23 Sep 2017 11:01:34 -0700 Subject: [PATCH 044/179] Bump Twisted requirement to 17.9.0 to catch many Python 3 fixes. --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index cc0a7f644..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 15.5.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From e733f51d4b04f209bfec32d1bd7559a258f45d0c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 25 Sep 2017 12:49:27 +0200 Subject: [PATCH 045/179] Fix test --- tests/test_command_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 2789d207c..4ac7fb786 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -28,4 +28,4 @@ class VersionTest(ProcessTest, unittest.TestCase): self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', - 'Platform']) + 'cryptography', 'Platform']) From d4555b2bcc387292e5fd5bd8321c946e2e374fb7 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 29 Sep 2017 12:07:29 +0900 Subject: [PATCH 046/179] update docs for supporting google cloud storage --- docs/topics/media-pipeline.rst | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index e948913a4..9580a15d9 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket) +* Specifying where to store the media (filesystem directory, Amazon S3 bucket, + Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: @@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: Supported Storage ================= -File system is currently the only officially supported storage, but there is -also support for storing files in `Amazon S3`_. +File system is currently the only officially supported storage, but there are +also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_. .. _Amazon S3: https://aws.amazon.com/s3/ +.. _Google Cloud Storage: https://cloud.google.com/storage/ File system storage ------------------- @@ -171,6 +173,25 @@ For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Google Cloud Storage +--------------------- + +.. setting:: GCS_PROJECT_ID + +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage +bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + +For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:: + + IMAGES_STORE = 'gs://bucket/images/' + GCS_PROJECT_ID = 'project_id' + +For information about authentication, see this `documentation`_. + +.. _documentation: https://cloud.google.com/docs/authentication/production + Usage example ============= From 59c3f6f095d7605a825a90f376d41493a78e5da7 Mon Sep 17 00:00:00 2001 From: Lucas Moauro Date: Sun, 1 Oct 2017 12:24:56 -0300 Subject: [PATCH 047/179] Fix typos in tests --- tests/test_downloadermiddleware_httpproxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 0ea83aaf9..17be875c1 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -28,7 +28,7 @@ class TestDefaultHeadersMiddleware(TestCase): crawler = Crawler(spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) - def test_no_enviroment_proxies(self): + def test_no_environment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -38,7 +38,7 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEqual(req.url, url) self.assertEqual(req.meta, {}) - def test_enviroment_proxies(self): + def test_environment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' os.environ.pop('file_proxy', None) From fc406801f1783392935fcd7faf603d8339c74675 Mon Sep 17 00:00:00 2001 From: Craig Rodrigues Date: Tue, 21 Mar 2017 00:21:41 -0700 Subject: [PATCH 048/179] ESMTPSenderFactory takes a message of bytes --- scrapy/mail.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 0bb395521..7365f25b7 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -5,10 +5,10 @@ See documentation in docs/topics/email.rst """ import logging -from six.moves import cStringIO as StringIO import six from email.utils import COMMASPACE, formatdate +from io import BytesIO from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase @@ -21,6 +21,14 @@ else: from twisted.internet import defer, reactor, ssl +try: + from twisted.mail.smtp import ESMTPSenderFactory +except ImportError: + """ + twisted.mail.smtp was not available in + older versions of Twisted on Python 3 + """ + from .utils.misc import arg_to_iter logger = logging.getLogger(__name__) @@ -110,9 +118,7 @@ class MailSender(object): 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - # Import twisted.mail here because it is not available in python3 - from twisted.mail.smtp import ESMTPSenderFactory - msg = StringIO(msg) + msg = BytesIO(msg.encode('utf-8')) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ From 12c7628fcbcfb3927f595e1fd6806ea6aefdf6fd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:13:45 +0200 Subject: [PATCH 049/179] Encode message using supplied charset --- scrapy/mail.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7365f25b7..7f237820f 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -5,10 +5,13 @@ See documentation in docs/topics/email.rst """ import logging +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO import six from email.utils import COMMASPACE, formatdate -from io import BytesIO from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase @@ -21,14 +24,6 @@ else: from twisted.internet import defer, reactor, ssl -try: - from twisted.mail.smtp import ESMTPSenderFactory -except ImportError: - """ - twisted.mail.smtp was not available in - older versions of Twisted on Python 3 - """ - from .utils.misc import arg_to_iter logger = logging.getLogger(__name__) @@ -96,7 +91,7 @@ class MailSender(object): 'mailattachs': len(attachs)}) return - dfd = self._sendmail(rcpts, msg.as_string()) + dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) dfd.addCallbacks(self._sent_ok, self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], errbackArgs=[to, cc, subject, len(attachs)]) @@ -118,7 +113,9 @@ class MailSender(object): 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - msg = BytesIO(msg.encode('utf-8')) + # Import twisted.mail here because it is not available in python3 + from twisted.mail.smtp import ESMTPSenderFactory + msg = BytesIO(msg) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ From 80bb4fcf9710598138d7604190636d723d6392df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:40:36 +0200 Subject: [PATCH 050/179] Convert SMTP credentials to bytes if needed --- scrapy/mail.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7f237820f..6d809dc99 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -24,7 +24,8 @@ else: from twisted.internet import defer, reactor, ssl -from .utils.misc import arg_to_iter +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) @@ -35,8 +36,8 @@ class MailSender(object): smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = smtpuser - self.smtppass = smtppass + self.smtpuser = to_bytes(smtpuser) + self.smtppass = to_bytes(smtppass) self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From 0d8a33fddccd7083c0f72fe75f89e9e7fd52cd82 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:42:56 +0200 Subject: [PATCH 051/179] Update docs --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index aac93a91a..949cdc638 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` setting will be used. If not given, no SMTP authentication will be performed. - :type smtphost: str + :type smtphost: str or bytes :param smtppass: the SMTP pass for authentication. - :type smtppass: str + :type smtppass: str or bytes :param smtpport: the SMTP port to connect to :type smtpport: int From 9cd348d94af782008f1a561a0b36da9231878833 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 22 Aug 2017 12:29:47 +0200 Subject: [PATCH 052/179] Handle None values for smtp user and password --- scrapy/mail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 6d809dc99..0cfb4ec79 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -36,8 +36,8 @@ class MailSender(object): smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = to_bytes(smtpuser) - self.smtppass = to_bytes(smtppass) + self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None + self.smtppass = to_bytes(smtppass) if smtppass is not None else None self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From f729d74886be2290fcbbcaa21d366b770ff21008 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 28 Aug 2017 11:21:11 +0200 Subject: [PATCH 053/179] Use a helper for to_bytes() and None input --- scrapy/mail.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 0cfb4ec79..5b944e1c4 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,14 +30,20 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +def _to_bytes_or_none(text): + if text is None: + return None + return to_bytes(text) + + class MailSender(object): def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None - self.smtppass = to_bytes(smtppass) if smtppass is not None else None + self.smtpuser = _to_bytes_or_none(smtpuser) + self.smtppass = _to_bytes_or_none(smtppass) self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From e914556adf8e556d4184db415c49266cc4c91bf5 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:12:01 +0200 Subject: [PATCH 054/179] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9bfdd803c..23343d92a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s).", + "max size (%(maxsize)s) in request %(request)s.", {'bytes': self._bytes_received, - 'maxsize': self._maxsize}) + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 938bc18405ca2cf60836bb8490c391c0fe445af1 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:31:00 +0200 Subject: [PATCH 055/179] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 23343d92a..0a5538947 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -344,8 +344,8 @@ class ScrapyAgent(object): if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s).", - {'size': expected_size, 'warnsize': warnsize}) + "download warn size (%(warnsize)s) in request (%(request)s).", + {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): # Abort connection inmediately. @@ -412,10 +412,9 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", + "max size (%(maxsize)s).", {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + 'maxsize': self._maxsize}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 345d948f2f55ad81ec7de7cb1f5619d80971a6c0 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:37:05 +0200 Subject: [PATCH 056/179] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0a5538947..4e1bb0cd5 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -344,7 +344,7 @@ class ScrapyAgent(object): if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s) in request (%(request)s).", + "download warn size (%(warnsize)s) in request %(request)s.", {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): From 9b8503011e1da0507e82e15631194ed99b7e699a Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Fri, 6 Oct 2017 13:45:35 +0200 Subject: [PATCH 057/179] Changed log message to include information about request as user djunzu commented --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 4e1bb0cd5..48d2481b4 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s).", + "max size (%(maxsize)s) in request %(request)s.", {'bytes': self._bytes_received, - 'maxsize': self._maxsize}) + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 9cdf34b7c791359b1f86678f758cf64368723c53 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 10 Oct 2017 22:49:22 +0530 Subject: [PATCH 058/179] Link "Debugging in Python" article to its new location Reference: https://web.archive.org/web/20170203104051/http://www.ferg.org/papers/debugging_in_python.html --- docs/topics/extensions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7a67cf295..c421a5e05 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. +:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT @@ -373,4 +373,4 @@ For more info see `Debugging in Python`. This extension only works on POSIX-compliant platforms (ie. not Windows). .. _Python debugger: https://docs.python.org/2/library/pdb.html -.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html +.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ From 8288f78a39939dbffea467bf110c64e005847117 Mon Sep 17 00:00:00 2001 From: djunzu Date: Mon, 16 Oct 2017 21:34:37 -0200 Subject: [PATCH 059/179] Add note about request.meta['depth'] in DepthMiddleware --- docs/topics/spider-middleware.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 9a0ccd0c1..a2d2556c5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -188,9 +188,13 @@ DepthMiddleware .. class:: DepthMiddleware - DepthMiddleware is a scrape middleware used for tracking the depth of each - Request inside the site being scraped. It can be used to limit the maximum - depth to scrape or things like that. + DepthMiddleware is used for tracking the depth of each Request inside the + site being scraped. It works by setting `request.meta['depth'] = 0` whenever + there is no value previously set (usually just the first Request) and + incrementing it by 1 otherwise. + + It can be used to limit the maximum depth to scrape, control Request + priority based on their depth, and things like that. The :class:`DepthMiddleware` can be configured through the following settings (see the settings documentation for more info): From 169dc2860e9f7054c50c84f5adcd8a0d5afe161e Mon Sep 17 00:00:00 2001 From: Weldon Malbrough Date: Mon, 16 Oct 2017 22:46:32 -0400 Subject: [PATCH 060/179] Update tutorial.rst startproject files Added middlewares.py to accurately reflect the file structure created by "scrapy startproject tutorial" --- docs/intro/tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 29f54bc86..a02c759bb 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file + + middlewares.py # project middlewares file pipelines.py # project pipelines file From 95815d27e89a6eea4676358697e2846959e4725b Mon Sep 17 00:00:00 2001 From: Weldon Malbrough Date: Wed, 25 Oct 2017 23:16:30 -0400 Subject: [PATCH 061/179] updated file structure to include middlewares.py --- docs/topics/commands.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index dc8067d7e..b15349598 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -55,6 +55,7 @@ structure by default, similar to this:: myproject/ __init__.py items.py + middlewares.py pipelines.py settings.py spiders/ From 9dd680d5c94340ac308f1450d9d3dc226a015326 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 26 Oct 2017 23:32:20 +0530 Subject: [PATCH 062/179] Use https for external links wherever possible in docs --- README.rst | 4 ++-- artwork/README.rst | 4 ++-- docs/contributing.rst | 2 +- docs/faq.rst | 2 +- docs/intro/install.rst | 12 ++++++------ docs/intro/overview.rst | 2 +- docs/intro/tutorial.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/deploy.rst | 2 +- docs/topics/firebug.rst | 2 +- docs/topics/firefox.rst | 8 ++++---- docs/topics/jobs.rst | 2 +- docs/topics/loaders.rst | 4 ++-- docs/topics/media-pipeline.rst | 2 +- docs/topics/practices.rst | 4 ++-- docs/topics/request-response.rst | 2 +- docs/topics/selectors.rst | 2 +- docs/topics/settings.rst | 4 ++-- docs/topics/shell.rst | 6 +++--- docs/topics/spiders.rst | 4 ++-- extras/coverage-report.sh | 2 +- 21 files changed, 37 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 27fab8e29..da63f2b93 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Scrapy :alt: PyPI Version .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: http://travis-ci.org/scrapy/scrapy + :target: https://travis-ci.org/scrapy/scrapy :alt: Build Status .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg @@ -15,7 +15,7 @@ Scrapy :alt: Wheel Status .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg - :target: http://codecov.io/github/scrapy/scrapy?branch=master + :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report .. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg diff --git a/artwork/README.rst b/artwork/README.rst index 016462f2c..92f6ecb7e 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -10,10 +10,10 @@ scrapy-logo.jpg Main Scrapy logo, in JPEG format. -qlassik.zip +qlassik.zip ----------- -Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font +Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font scrapy-blog.logo.xcf -------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index c969bd842..291a1054e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -188,7 +188,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: http://reddit.com/r/scrapy +.. _Scrapy subreddit: https://reddit.com/r/scrapy .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests diff --git a/docs/faq.rst b/docs/faq.rst index f0ee20b5e..42c3abbfa 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -21,7 +21,7 @@ Python code. In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like comparing `jinja2`_ to `Django`_. -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _jinja2: http://jinja.pocoo.org/ .. _Django: https://www.djangoproject.com/ diff --git a/docs/intro/install.rst b/docs/intro/install.rst index deb0118d4..12d489612 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -188,7 +188,7 @@ solutions: that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: - * Install `homebrew`_ following the instructions in http://brew.sh/ + * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly @@ -233,9 +233,9 @@ After any of these workarounds you should be able to install Scrapy:: .. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ -.. _homebrew: http://brew.sh/ -.. _zsh: http://www.zsh.org/ +.. _homebrew: https://brew.sh/ +.. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com -.. _Anaconda: http://docs.continuum.io/anaconda/index -.. _Miniconda: http://conda.pydata.org/docs/install/quick.html -.. _conda-forge: https://conda-forge.github.io/ +.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Miniconda: https://conda.io/docs/user-guide/install/index.html +.. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 1da1a4059..d0ce07a8e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -164,4 +164,4 @@ interest! .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html .. _Amazon S3: https://aws.amazon.com/s3/ -.. _Sitemaps: http://www.sitemaps.org +.. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a02c759bb..20538e90f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -34,7 +34,7 @@ list of Python resources for non-programmers`_. .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial -.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ +.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/ Creating a project diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index a3e72097c..d1991c02f 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -142,4 +142,4 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. -.. _base tag: http://www.w3schools.com/tags/tag_base.asp +.. _base tag: https://www.w3schools.com/tags/tag_base.asp diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index f2e11fe8f..361914a29 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -55,5 +55,5 @@ just like ``scrapyd-deploy``. .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html +.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html .. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 8f0a5767b..4ea8d3bd0 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the Project`_ used in the :ref:`tutorial ` but with a different face. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Google Directory: http://directory.google.com/ .. _Open Directory Project: http://www.dmoz.org diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst index 0cf45861a..2c85848be 100644 --- a/docs/topics/firefox.rst +++ b/docs/topics/firefox.rst @@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one after applying some browser clean up and executing Javascript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. +extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things when working with Firefox and XPath: @@ -71,11 +71,11 @@ Firecookie `Firecookie`_ makes it easier to view and manage cookies. You can use this extension to create a new cookie, delete existing cookies, see a list of cookies -for the current site, manage cookies permissions and a lot more. +for the current site, manage cookies permissions and a lot more. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA -.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ +.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ .. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/ .. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/ .. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/ diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 4f9e38086..06c7fff3d 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. -.. _pickle: http://docs.python.org/library/pickle.html +.. _pickle: https://docs.python.org/library/pickle.html diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index ad86dba63..0849090b4 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -518,8 +518,8 @@ a footer of a page that looks something like: Example:: diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 9580a15d9..4c634ace5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -171,7 +171,7 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Google Cloud Storage --------------------- diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 21aa4a0a7..e0dd4000f 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -249,9 +249,9 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Tor project: https://www.torproject.org/ .. _commercial support: http://scrapy.org/support/ -.. _ProxyMesh: http://proxymesh.com/ +.. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: https://scrapinghub.com/crawlera -.. _scrapoxy: http://scrapoxy.io/ +.. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 92aae1ad0..121abe6b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -734,7 +734,7 @@ HtmlResponse objects which adds encoding auto-discovering support by looking into the HTML `meta http-equiv`_ attribute. See :attr:`TextResponse.encoding`. -.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp +.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp XmlResponse objects ------------------- diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9bae53f45..cb4c25391 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -36,7 +36,7 @@ documents. For a complete reference of the selectors API see :ref:`Selector reference ` -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html .. _cssselect: https://pypi.python.org/pypi/cssselect/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 37e3828a4..4b15cb607 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1002,7 +1002,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option. If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. -.. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. _wget: https://www.gnu.org/software/wget/manual/wget.html .. setting:: REACTOR_THREADPOOL_MAXSIZE @@ -1317,7 +1317,7 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html .. setting:: USER_AGENT diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index ef6aeeed3..527116418 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: [settings] shell = bpython -.. _IPython: http://ipython.org/ -.. _IPython installation guide: http://ipython.org/install.html -.. _bpython: http://www.bpython-interpreter.org/ +.. _IPython: https://ipython.org/ +.. _IPython installation guide: https://ipython.org/install.html +.. _bpython: https://www.bpython-interpreter.org/ Launch the shell ================ diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index bf1532d1b..c2c271245 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -751,8 +751,8 @@ Combine SitemapSpider with other sources of urls:: def parse_other(self, response): pass # ... scrape other here ... -.. _Sitemaps: http://www.sitemaps.org -.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index +.. _Sitemaps: https://www.sitemaps.org/index.html +.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: http://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh index dc20e16e4..842d0e46e 100755 --- a/extras/coverage-report.sh +++ b/extras/coverage-report.sh @@ -1,6 +1,6 @@ # Run tests, generate coverage report and open it on a browser # -# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage +# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage coverage run --branch $(which trial) --reporter=text tests coverage html -i From 9d9d83a8c31b6a18d7aaac35a30ffb69db4bb81d Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:24:40 +0530 Subject: [PATCH 063/179] Use https links wherever possible --- CONTRIBUTING.md | 4 ++-- INSTALL | 2 +- README.rst | 14 ++++++------- debian/control | 6 +++--- debian/copyright | 8 ++++---- docs/contributing.rst | 2 +- docs/intro/overview.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/selectors.rst | 4 ++-- docs/topics/shell.rst | 8 ++++---- scrapy/_monkeypatches.py | 4 ++-- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/chunked.py | 2 +- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/exporters.py | 2 +- scrapy/extensions/httpcache.py | 10 +++++----- scrapy/extensions/telnet.py | 2 +- scrapy/pipelines/files.py | 4 ++-- scrapy/signalmanager.py | 2 +- scrapy/templates/project/module/items.py.tmpl | 2 +- .../project/module/middlewares.py.tmpl | 2 +- .../project/module/pipelines.py.tmpl | 2 +- .../templates/project/module/settings.py.tmpl | 20 +++++++++---------- scrapy/utils/defer.py | 2 +- scrapy/utils/deprecate.py | 12 +++++------ scrapy/utils/http.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/url.py | 2 +- sep/sep-001.rst | 2 +- sep/sep-006.rst | 4 ++-- sep/sep-013.rst | 2 +- sep/sep-017.rst | 2 +- sep/sep-020.rst | 2 +- setup.py | 2 +- tests/__init__.py | 2 +- tests/keys/example-com.conf | 4 ++-- 37 files changed, 76 insertions(+), 76 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88c472f6f..0a11b05d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ The guidelines for contributing are available here: -http://doc.scrapy.org/en/master/contributing.html +https://doc.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the -support channels to get it answered: http://scrapy.org/community/ +support channels to get it answered: https://scrapy.org/community/ diff --git a/INSTALL b/INSTALL index 84803a933..a3c7899c6 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* http://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://doc.scrapy.org/en/latest/intro/install.html (online version) diff --git a/README.rst b/README.rst index da63f2b93..45135c7a2 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. For more information including a list of features check the Scrapy homepage at: -http://scrapy.org +https://scrapy.org Requirements ============ @@ -47,12 +47,12 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -http://doc.scrapy.org/en/latest/intro/install.html +https://doc.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at http://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` directory. Releases @@ -63,12 +63,12 @@ You can find release notes at https://doc.scrapy.org/en/latest/news.html Community (blog, twitter, mail list, IRC) ========================================= -See http://scrapy.org/community/ +See https://scrapy.org/community/ Contributing ============ -See http://doc.scrapy.org/en/master/contributing.html +See https://doc.scrapy.org/en/master/contributing.html Code of Conduct --------------- @@ -82,9 +82,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com. Companies using Scrapy ====================== -See http://scrapy.org/companies/ +See https://scrapy.org/companies/ Commercial Support ================== -See http://scrapy.org/support/ +See https://scrapy.org/support/ diff --git a/debian/control b/debian/control index f3a31753b..2cc8eedf4 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: Scrapinghub Team Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) Standards-Version: 3.8.4 -Homepage: http://scrapy.org/ +Homepage: https://scrapy.org/ Package: scrapy Architecture: all @@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25 Provides: python-scrapy, scrapy-0.25 Description: Python web crawling and web scraping framework Scrapy is a fast high-level web crawling and web scraping framework, - used to crawl websites and extract structured data from their pages. - It can be used for a wide range of purposes, from data mining to + used to crawl websites and extract structured data from their pages. + It can be used for a wide range of purposes, from data mining to monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright index 4cc239002..c1bf47565 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,6 +1,6 @@ This package was debianized by the Scrapinghub team . -It was downloaded from http://scrapy.org +It was downloaded from https://scrapy.org Upstream Author: Scrapy Developers @@ -14,10 +14,10 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. diff --git a/docs/contributing.rst b/docs/contributing.rst index 291a1054e..f3732ab06 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,7 +7,7 @@ Contributing to Scrapy .. important:: Double check you are reading the most recent version of this document at - http://doc.scrapy.org/en/master/contributing.html + https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d0ce07a8e..6f1c2c43f 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -160,7 +160,7 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: http://scrapy.org/community/ +.. _join the community: https://scrapy.org/community/ .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html .. _Amazon S3: https://aws.amazon.com/s3/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index e0dd4000f..02cfa9b05 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -248,7 +248,7 @@ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. .. _Tor project: https://www.torproject.org/ -.. _commercial support: http://scrapy.org/support/ +.. _commercial support: https://scrapy.org/support/ .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cb4c25391..8ac40c3cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy documentation server: - http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://doc.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -99,7 +99,7 @@ Here's its HTML code: First, let's open the shell:: - scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 527116418..11ab199f2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -142,7 +142,7 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then proceed to scrape the https://reddit.com +https://scrapy.org page, and then proceed to scrape the https://reddit.com page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. @@ -154,7 +154,7 @@ shell works. First, we launch the shell:: - scrapy shell 'http://scrapy.org' --nolog + scrapy shell 'https://scrapy.org' --nolog Then, the shell fetches the URL (using the Scrapy downloader) and prints the list of available objects and useful shortcuts (you'll notice that these lines @@ -164,7 +164,7 @@ all start with the ``[s]`` prefix):: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request + [s] request [s] response <200 https://scrapy.org/> [s] settings [s] spider @@ -182,7 +182,7 @@ After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("http://reddit.com") + >>> fetch("https://reddit.com") >>> response.xpath('//title/text()').extract() ['reddit: the front page of the internet'] diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 60e0de1f2..f55ecc213 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -4,12 +4,12 @@ from six.moves import copyreg if sys.version_info[0] == 2: from urlparse import urlparse - # workaround for http://bugs.python.org/issue7904 - Python < 2.7 + # workaround for https://bugs.python.org/issue7904 - Python < 2.7 if urlparse('s3://bucket/key').netloc != 'bucket': from urlparse import uses_netloc uses_netloc.append('s3') - # workaround for http://bugs.python.org/issue9374 - Python < 2.7.4 + # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 if urlparse('s3://bucket/key?key=value').query != 'key=value': from urlparse import uses_query uses_query.append('s3') diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a94a89205..783d4c383 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -64,7 +64,7 @@ if twisted_version >= (14, 0, 0): """ Twisted-recommended context factory for web clients. - Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: "The default is to use a BrowserLikePolicyForHTTPS, so unless you have special requirements you can leave this as-is." @@ -100,6 +100,6 @@ else: def getContext(self, hostname=None, port=None): ctx = ClientContextFactory.getContext(self) # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html ctx.set_options(SSL.OP_ALL) return ctx diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a33ce9805..5cbc2d7c5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -83,7 +83,7 @@ class Crawler(object): yield defer.maybeDeferred(self.engine.start) except Exception: # In Python 2 reraising an exception after yield discards - # the original traceback (see http://bugs.python.org/issue7563), + # the original traceback (see https://bugs.python.org/issue7563), # so sys.exc_info() workaround is used. # This workaround also works in Python 3, but it is not needed, # and it is slower, so in Python 3 we use native `raise`. diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 64d94c489..6748d0265 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -11,7 +11,7 @@ warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as - documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding + documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ def process_response(self, request, response, spider): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..495b103d1 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -75,7 +75,7 @@ class HttpCacheMiddleware(object): return response # RFC2616 requires origin server to set Date header, - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: response.headers['Date'] = formatdate(usegmt=1) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index e2d42b6ab..07f43b494 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -188,7 +188,7 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(name) self._beautify_newline() - # Workaround for http://bugs.python.org/issue17606 + # Workaround for https://bugs.python.org/issue17606 # Before Python 2.7.4 xml.sax.saxutils required bytes; # since 2.7.4 it requires unicode. The bug is likely to be # fixed in 2.7.6, but 2.7.6 will still support unicode, diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 648b32ec7..1b5e05b1b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -70,8 +70,8 @@ class RFC2616Policy(object): return True def should_cache_response(self, response, request): - # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 - # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" @@ -163,7 +163,7 @@ class RFC2616Policy(object): def _compute_freshness_lifetime(self, response, request, now): # Reference nsHttpResponseHead::ComputeFreshnessLifetime - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) maxage = self._get_max_age(cc) if maxage is not None: @@ -194,7 +194,7 @@ class RFC2616Policy(object): def _compute_current_age(self, response, request, now): # Reference nsHttpResponseHead::ComputeCurrentAge - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#366 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server @@ -414,7 +414,7 @@ class LeveldbCacheStorage(object): def parse_cachecontrol(header): """Parse Cache-Control header - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None, ... b'max-age': b'3600'} diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index d9add1d97..5ca0d19a0 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -82,7 +82,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " \ - "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 7fdb8a086..9f1faa313 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -120,7 +120,7 @@ class S3FilesStore(object): def _get_boto_bucket(self): # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 + # https://bugs.python.org/issue5103 c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) return c.get_bucket(self.bucket, validate=False) @@ -268,7 +268,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index fd79905e9..296d27ed8 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -55,7 +55,7 @@ class SignalManager(object): The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html + .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 2c746138f..7d766f4fc 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/items.html +# https://doc.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 1a4b0caa5..c5b542bd6 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4e9b32e9e..e58dab089 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 35a0f9a45..a0557473e 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# http://doc.scrapy.org/en/latest/topics/settings.html -# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html -# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html +# See https://doc.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See http://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bb4c74a6e..aa6dcffda 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -57,7 +57,7 @@ def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. - Taken from: http://jcalderone.livejournal.com/24285.html + Taken from: https://jcalderone.livejournal.com/24285.html """ coop = task.Cooperator() work = (callable(elem, *args, **named) for elem in iterable) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 7ab39c97e..f76161a68 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -71,8 +71,8 @@ def create_deprecated_class(name, new_class, clsdict=None, warnings.warn(msg, warn_category, stacklevel=2) super(DeprecatedClass, cls).__init__(name, bases, clsdict_) - # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass - # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks + # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass + # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) @@ -159,10 +159,10 @@ def update_classpath(path): def method_is_overridden(subclass, base_class, method_name): - """ - Return True if a method named ``method_name`` of a ``base_class`` - is overridden in a ``subclass``. - + """ + Return True if a method named ``method_name`` of a ``base_class`` + is overridden in a ``subclass``. + >>> class Base(object): ... def foo(self): ... pass diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index 8b659a22a..7cc8d1884 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -11,7 +11,7 @@ def decode_chunked_transfer(chunked_body): decoded body. For more info see: - http://en.wikipedia.org/wiki/Chunked_transfer_encoding + https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ body, h, t = '', '', chunked_body diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 7c95e1e50..828880709 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -154,7 +154,7 @@ class StreamLogger(object): """Fake file-like stream object that redirects writes to a logger instance Taken from: - http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ + https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ def __init__(self, logger, log_level=logging.INFO): self.logger = logger diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 8eed31060..657c53815 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -47,7 +47,7 @@ def parse_url(url, encoding=None): def escape_ajax(url): """ Return the crawleable url according to: - http://code.google.com/web/ajaxcrawling/docs/getting-started.html + https://developers.google.com/webmasters/ajax-crawling/docs/getting-started >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 2f0fe3500..3766f38fc 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see http://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index c0f945b66..522bba134 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -16,7 +16,7 @@ Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [http://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -71,5 +71,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (http://doc.scrapy.org/topics/selectors.html) + 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html) diff --git a/sep/sep-013.rst b/sep/sep-013.rst index 4c11a0762..5b18b7501 100644 --- a/sep/sep-013.rst +++ b/sep/sep-013.rst @@ -44,7 +44,7 @@ Overview of changes proposed Most of the inconsistencies come from the fact that middlewares don't follow the typical -[http://twistedmatrix.com/projects/core/documentation/howto/defer.html +[https://twistedmatrix.com/projects/core/documentation/howto/defer.html deferred] callback/errback chaining logic. Twisted logic is fine and quite intuitive, and also fits middlewares very well. Due to some bad design choices the integration between middleware calls and deferred is far from optional. So diff --git a/sep/sep-017.rst b/sep/sep-017.rst index 7707a1622..86005e3c9 100644 --- a/sep/sep-017.rst +++ b/sep/sep-017.rst @@ -13,7 +13,7 @@ SEP-017: Spider Contracts The motivation for Spider Contracts is to build a lightweight mechanism for testing your spiders, and be able to run the tests quickly without having to wait for all the spider to run. It's partially based on the -[http://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach +[https://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach (hence its name) where you define certain conditions that spider callbacks must met, and you give example testing pages. diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 49d068479..52d78097b 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -29,7 +29,7 @@ the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader -is the `definition list `_:: +is the `definition list `_::
diff --git a/setup.py b/setup.py index c03f0b9f7..327286f5a 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ if has_environment_marker_platform_impl_support(): setup( name='Scrapy', version=version, - url='http://scrapy.org', + url='https://scrapy.org', description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst').read(), author='Scrapy developers', diff --git a/tests/__init__.py b/tests/__init__.py index c2e4fd2bf..55b1ecde8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see http://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://doc.scrapy.org/en/latest/contributing.html#running-tests """ import os diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf index 8aa338cd5..1f9c25e43 100644 --- a/tests/keys/example-com.conf +++ b/tests/keys/example-com.conf @@ -1,4 +1,4 @@ -# this is copied from http://stackoverflow.com/a/27931596 +# this is copied from https://stackoverflow.com/a/27931596 [ req ] default_bits = 2048 default_keyfile = server-key.pem @@ -24,7 +24,7 @@ organizationName_default = Example, LLC # Use a friendly name here because its presented to the user. The server's DNS # names are placed in Subject Alternate Names. Plus, DNS names here is deprecated -# by both IETF and CA/Browser Forums. If you place a DNS name here, then you +# by both IETF and CA/Browser Forums. If you place a DNS name here, then you # must include the DNS name in the SAN too (otherwise, Chrome and others that # strictly follow the CA/Browser Baseline Requirements will fail). commonName = Common Name (e.g. server FQDN or YOUR name) From 23c7437e4629199e8ee1ae6bcdf75b7062466010 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:34:49 +0530 Subject: [PATCH 064/179] Fix link for 'XPath and XSLT with lxml' --- sep/sep-006.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 522bba134..7425c0930 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -58,7 +58,7 @@ As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to `select`, `sel` (is shortness if required), or `xpath` after `lxml's -`_ ``xpath`` method. +`_ ``xpath`` method. Bonus (ItemBuilder) =================== @@ -72,4 +72,4 @@ References ========== 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) - 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html) + 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) From 97d047a055b3af080047768b196ce677fbfaa12e Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:48:41 +0530 Subject: [PATCH 065/179] Fix link for Tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0608693ba..f35b894f3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests +# Tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. From dae7b1cdd06649db3c692eccd70195a499733448 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:53:32 +0530 Subject: [PATCH 066/179] Migrate all subdomains on readthedocs.org to readthedocs.io --- scrapy/templates/project/scrapy.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg index d7f02e0a2..1daeaa541 100644 --- a/scrapy/templates/project/scrapy.cfg +++ b/scrapy/templates/project/scrapy.cfg @@ -1,7 +1,7 @@ # Automatically created by: scrapy startproject # # For more information about the [deploy] section see: -# https://scrapyd.readthedocs.org/en/latest/deploy.html +# https://scrapyd.readthedocs.io/en/latest/deploy.html [settings] default = ${project_name}.settings From df7e0a4315f9db2c74fa9e9a0654f44277da2e55 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 23:37:44 +0530 Subject: [PATCH 067/179] Use https link in default user agent --- docs/topics/settings.rst | 2 +- scrapy/settings/default_settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b15cb607..afa666659 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1324,7 +1324,7 @@ the default value for this setting see: https://boutell.com/newfaq/misc/urllengt USER_AGENT ---------- -Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 697314b7f..ead511473 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -270,7 +270,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] From 8a7552370de2d31fbd3564e47fa92049c8efee6f Mon Sep 17 00:00:00 2001 From: colinmorris Date: Tue, 31 Oct 2017 17:14:53 -0400 Subject: [PATCH 068/179] revise/modernize item exporter example in docs --- docs/topics/exporters.rst | 50 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index b6139af92..28f5ad9c2 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -36,38 +36,36 @@ to export 3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal the end of the exporting process -Here you can see an :doc:`Item Pipeline ` which uses an Item -Exporter to export scraped items to different files, one per spider:: +Here you can see an :doc:`Item Pipeline ` which uses multiple +Item Exporters to group scraped items to different files according to the +value of one of their fields:: - from scrapy import signals - from scrapy.exporters import XmlItemExporter + from scrapy.exporters import XmlItemExporter - class XmlExportPipeline(object): + class PerYearXmlExportPipeline(object): + """Distribute items across multiple XML files according to their 'year' field""" - def __init__(self): - self.files = {} + def open_spider(self, spider): + self.year_to_exporter = {} - @classmethod - def from_crawler(cls, crawler): - pipeline = cls() - crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) - crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) - return pipeline + def close_spider(self, spider): + for exporter in self.year_to_exporter.itervalues(): + exporter.finish_exporting() + exporter.file.close() - def spider_opened(self, spider): - file = open('%s_products.xml' % spider.name, 'w+b') - self.files[spider] = file - self.exporter = XmlItemExporter(file) - self.exporter.start_exporting() + def _exporter_for_item(self, item): + year = item['year'] + if year not in self.year_to_exporter: + f = open('{}.xml'.format(year), 'w+b') + exporter = XmlItemExporter(f) + exporter.start_exporting() + self.year_to_exporter[year] = exporter + return self.year_to_exporter[year] - def spider_closed(self, spider): - self.exporter.finish_exporting() - file = self.files.pop(spider) - file.close() - - def process_item(self, item, spider): - self.exporter.export_item(item) - return item + def process_item(self, item, spider): + exporter = self._exporter_for_item(item) + exporter.export_item(item) + return item .. _topics-exporters-field-serialization: From 23e571e860729fad1f4351cde69b77d88837e628 Mon Sep 17 00:00:00 2001 From: colinmorris Date: Tue, 31 Oct 2017 18:08:47 -0400 Subject: [PATCH 069/179] fix issues identified in review --- docs/topics/exporters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 28f5ad9c2..95f7920f8 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -49,14 +49,14 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.itervalues(): + for exporter in self.year_to_exporter.values(): exporter.finish_exporting() exporter.file.close() def _exporter_for_item(self, item): year = item['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'w+b') + f = open('{}.xml'.format(year), 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter From abb6d0a1c1dd4d1ab6f734f2b366b9d851a52a48 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 1 Nov 2017 17:26:59 +0300 Subject: [PATCH 070/179] Use portable pypy directly They are provided by https://github.com/squeaky-pl/portable-pypy --- .travis.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c51fafb2..d4f30814d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,10 @@ matrix: install: - | if [ "$TOXENV" = "pypy" ]; then - export PYENV_ROOT="$HOME/.pyenv" - if [ -f "$PYENV_ROOT/bin/pyenv" ]; then - pushd "$PYENV_ROOT" && git pull && popd - else - rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" - fi - # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1` - "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" - virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - pip install -U tox twine wheel codecov From 846fd83512bb45335195b060d76b46060b4b6e3d Mon Sep 17 00:00:00 2001 From: IAlwaysBeCoding Date: Sat, 11 Nov 2017 18:30:01 -0500 Subject: [PATCH 071/179] removed commented out code, wrapped line to pep-8 and removed backlashes --- docs/topics/commands.rst | 3 +++ scrapy/commands/parse.py | 23 ++++++++++++++++++++++ tests/test_command_parse.py | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index dc8067d7e..07c69ddda 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -430,6 +430,9 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response +* ``--meta`` or ``-m``: additional request meta that will be pass to the callback + request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a90095146..69418a478 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,5 @@ from __future__ import print_function +import json import logging from w3lib.url import is_url @@ -48,6 +49,8 @@ class Command(ScrapyCommand): help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", help="use this callback for parsing, instead looking for a callback") + parser.add_option("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", @@ -204,6 +207,10 @@ class Command(ScrapyCommand): req.callback = callback return requests + #update request meta if any extra meta was passed through the --meta/-m opts. + if opts.meta: + request.meta.update(opts.meta) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -211,11 +218,27 @@ class Command(ScrapyCommand): def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) + + self.process_spider_arguments(opts) + self.process_request_meta(opts) + + def process_spider_arguments(self, opts): + try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + def process_request_meta(self, opts): + + if opts.meta: + try: + opts.meta = json.loads(opts.meta) + except ValueError: + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + + def run(self, args, opts): # parse arguments if not len(args) == 1 or not is_url(args[0]): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b6d6db9ee..66dd17110 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,6 +29,21 @@ class MySpider(scrapy.Spider): self.logger.debug('It Works!') return [scrapy.Item(), dict(foo='bar')] + def parse_request_with_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Does Not Work :(') + else: + self.logger.debug('It Works!') + + def parse_request_without_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{0}' @@ -84,6 +99,30 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + @defer.inlineCallbacks + def test_request_with_meta(self): + raw_json_string = '{"foo" : "baz"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--meta', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-m', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + + @defer.inlineCallbacks + def test_request_without_meta(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-c', 'parse_request_without_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, From 62a626102877de4998538717f34e61d2f7d2622c Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sat, 18 Nov 2017 20:03:59 +0100 Subject: [PATCH 072/179] Issues a warning when user puts a URL into allowed_domains (#2250) --- scrapy/spidermiddlewares/offsite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index ea1c9270f..f51b0a2b0 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,6 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + for domainIndex in range(0, len(allowed_domains)): + url_pattern = re.compile("^https?://.*$") + if url_pattern.match(allowed_domains[domainIndex]): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) From 5441cc18e43cba6b8196ece49d6b454badb246f0 Mon Sep 17 00:00:00 2001 From: KosayJabre Date: Sun, 19 Nov 2017 18:09:38 -0400 Subject: [PATCH 073/179] Separated import statements Just separated the import statements. Tiny change - testing GitHub! --- scrapy/commands/edit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index a7f8983b4..25d843a53 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,4 +1,5 @@ -import sys, os +import sys +import os from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError From 91ff194d1e9477d2196817ea1dc8beb220c3e058 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Mon, 20 Nov 2017 21:23:31 +0100 Subject: [PATCH 074/179] looping over allowed_domains directly instead of via index --- scrapy/spidermiddlewares/offsite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f51b0a2b0..8ff35e29f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,10 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - for domainIndex in range(0, len(allowed_domains)): + for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") - if url_pattern.match(allowed_domains[domainIndex]): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) + if url_pattern.match(domain): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) From 0b14cb44aabcc8322e09a46c4e14913c94886e12 Mon Sep 17 00:00:00 2001 From: Jesse Bakker Date: Thu, 23 Nov 2017 15:25:43 +0100 Subject: [PATCH 075/179] Added from_crawler to middleware docs --- docs/topics/downloader-middleware.rst | 11 +++++++++++ docs/topics/spider-middleware.rst | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0d168017f..983a93290 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -157,6 +157,17 @@ more of the following methods: :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a2d2556c5..c297ed556 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -164,6 +164,17 @@ following methods: :param spider: the spider to whom the start requests belong :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception From 6af323d7c85eeee40d90a20133504df26a593304 Mon Sep 17 00:00:00 2001 From: IAlwaysBeCoding Date: Sun, 26 Nov 2017 00:24:52 +0100 Subject: [PATCH 076/179] Fix spelling mistake on scrapy parse command docs Fixed spelling mistake from "will be pass" to "will be passed" --- docs/topics/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 07c69ddda..06f9a485b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -430,7 +430,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be pass to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--pipelines``: process items through pipelines From 8ec3b476b03d6b8424f6dfc556758392e7a5a61f Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 16:36:15 +0100 Subject: [PATCH 077/179] triggering a warning when user puts URL in allowed_domains now covered by test --- scrapy/spidermiddlewares/offsite.py | 3 ++- tests/test_spidermiddleware_offsite.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 8ff35e29f..647792e5d 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst import re import logging +import warnings from scrapy import signals from scrapy.http import Request @@ -55,7 +56,7 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 9ad86313c..b532cc2ec 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -6,6 +6,7 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler +import warnings class TestOffsiteMiddleware(TestCase): @@ -68,3 +69,13 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + + +class TestOffsiteMiddleware5(TestOffsiteMiddleware4): + + def test_get_host_regex(self): + self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.get_host_regex(self.spider) + assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message) From 454d5e57333e9f33c8d684e4e21f8f7e9493f310 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 20:07:04 +0100 Subject: [PATCH 078/179] checking for subclass of URLWarning instead of checking error message text when URL in allowed_domains --- scrapy/spidermiddlewares/offsite.py | 7 ++++++- tests/test_spidermiddleware_offsite.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 647792e5d..f595eef42 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -56,10 +56,15 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) def spider_opened(self, spider): self.host_regex = self.get_host_regex(spider) self.domains_seen = set() + + +class URLWarning(Warning): + pass \ No newline at end of file diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b532cc2ec..7e4af0d4c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -5,6 +5,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler import warnings @@ -78,4 +79,4 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.get_host_regex(self.spider) - assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message) + assert issubclass(w[-1].category, URLWarning) From 22c68baf990f15d249f38c481f24a984977be3e5 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Thu, 7 Dec 2017 18:38:29 +0100 Subject: [PATCH 079/179] url_pattern is now being compiled before entering the loop --- scrapy/spidermiddlewares/offsite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f595eef42..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -53,8 +53,8 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: - url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) @@ -67,4 +67,4 @@ class OffsiteMiddleware(object): class URLWarning(Warning): - pass \ No newline at end of file + pass From f716843a66829350063e55f8df768eb538c6b05c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 12 Dec 2017 16:19:43 +0500 Subject: [PATCH 080/179] DOC update "Contributing" docs: * suggest Stack Overflow for Scrapy usage questions; * encourage users to submit test-only pull requests with reproducable examples; * encourage users to pick up stalled pull requests; * we don't use AUTHORS file as a main acknowledgement source; * suggest using Sphinx autodocs extension --- docs/contributing.rst | 58 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f3732ab06..eb736bf30 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -19,12 +19,16 @@ There are many ways to contribute to Scrapy. Here are some of them: the guidelines detailed in `Reporting bugs`_ below. * Submit patches for new functionality and/or bug fixes. Please read - `Writing patches`_ and `Submitting patches`_ below for details on how to + :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. * Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. +* Answer Scrapy questions at + `Stack Overflow `__. + + Reporting bugs ============== @@ -40,9 +44,14 @@ guidelines when reporting a new bug. * check the :ref:`FAQ ` first to see if your issue is addressed in a well-known question +* if you have a general question about scrapy usage, please ask it at + `Stack Overflow `__ + (use "scrapy" tag). + * check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report but check the ticket history and comments, you may - find additional useful information to contribute. + don't dismiss the report, but check the ticket history and comments. If you + have additional useful information, please leave a comment, or consider + :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has been discussed there, or if you're not sure if what you're seeing is a bug. @@ -54,12 +63,20 @@ guidelines when reporting a new bug. it. See for example StackOverflow's guide on creating a `Minimal, Complete, and Verifiable example`_ exhibiting the issue. +* the most awesome way to provide a complete reproducible example is to + send a pull request which adds a failing test case to the + Scrapy testing suite (see :ref:`submitting-patches`). + This is helpful even if you don't have an intention to + fix the issue yourselves. + * include the output of ``scrapy version -v`` so developers working on your bug know exactly which version and platform it occurred on, which is often very helpful for reproducing it, or knowing if it was already fixed. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _writing-patches: + Writing patches =============== @@ -83,6 +100,8 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +.. _submitting-patches: + Submitting patches ================== @@ -100,11 +119,22 @@ starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea has been validated and proven useful. Alternatively, you can start a conversation in the `Scrapy subreddit`_ to discuss your idea first. + +Sometimes there is an existing pull request for the problem you'd like to +solve, which is stalled for some reason. Often the pull request is in a +right direction, but changes are requested by Scrapy maintainers, and the +original pull request author haven't had time to address them. +In this case consider picking up this pull request: open +a new pull request with all commits from the original pull request, as well as +additional changes to address the raised issues. Doing so helps a lot; it is +not considered rude as soon as original the author is acknowledged by keeping +his/her commits. + When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" -instead of "Fix for #411". -Complete titles make it easy to skim through the issue tracker. +instead of "Fix for #411". Complete titles make it easy to skim through +the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits than functional changes. This will make pull @@ -121,21 +151,29 @@ Scrapy: * It's OK to use lines longer than 80 chars if it improves the code readability. -* Don't put your name in the code you contribute. Our policy is to keep - the contributor's name in the `AUTHORS`_ file distributed with Scrapy. +* Don't put your name in the code you contribute; git provides enough + metadata to identify author of the code. + See https://help.github.com/articles/setting-your-username-in-git/ for + setup instructions. Documentation policies ====================== * **Don't** use docstrings for documenting classes, or methods which are - already documented in the official (sphinx) documentation. For example, the - :meth:`ItemLoader.add_value` method should be documented in the sphinx - documentation, not its docstring. + already documented in the official (sphinx) documentation. Alternatively, + **do** provide a docstring, but make sure sphinx documentation uses + autodoc_ extension to pull the docstring. For example, the + :meth:`ItemLoader.add_value` method should be either + documented only in the sphinx documentation (not it a docstring), or + it should have a docstring which is pulled to sphinx documentation using + autodoc_ extension. * **Do** use docstrings for documenting functions not present in the official (sphinx) documentation, such as functions from ``scrapy.utils`` package and its sub-modules. +.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html + Tests ===== From 9aa9dd8d45a2ce0c8e6ae0732e610f020735df7e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 12 Dec 2017 19:17:00 +0500 Subject: [PATCH 081/179] DOC mention an easier way to track pull requests locally. Thanks @eliasdorneles! --- docs/contributing.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index eb736bf30..9a02634cb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -127,9 +127,16 @@ original pull request author haven't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as original the author is acknowledged by keeping +not considered rude as soon as the original author is acknowledged by keeping his/her commits. +You can pull an existing pull request to a local branch +by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` +(replace 'upstream' with a remote name for scrapy repository, +``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` +with a name of the branch you want to create locally). +See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally. + When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" From 44623687ab8936c5696f68f74e438a2891880c82 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 19 Dec 2017 17:59:05 +0200 Subject: [PATCH 082/179] Drop support for EOL Python 3.3 --- .travis.yml | 2 -- README.rst | 6 +++++- docs/faq.rst | 2 +- docs/intro/install.rst | 10 +++++----- setup.py | 2 +- tox.ini | 12 ++++-------- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4f30814d..66de9ed03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,6 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy - - python: 3.3 - env: TOXENV=py33 - python: 3.5 env: TOXENV=py35 - python: 3.6 diff --git a/README.rst b/README.rst index 45135c7a2..1361eac26 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,10 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: PyPI Version +.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg + :target: https://pypi.python.org/pypi/Scrapy + :alt: Supported Python Versions + .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg :target: https://travis-ci.org/scrapy/scrapy :alt: Build Status @@ -36,7 +40,7 @@ https://scrapy.org Requirements ============ -* Python 2.7 or Python 3.3+ +* Python 2.7 or Python 3.4+ * Works on Linux, Windows, Mac OSX, BSD Install diff --git a/docs/faq.rst b/docs/faq.rst index 42c3abbfa..484226979 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.3+. +Scrapy is supported under Python 2.7 and Python 3.4+. Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 12d489612..a2e3f506e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above. +Scrapy runs on Python 2.7 and Python 3.4 or above. If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -132,12 +132,12 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: .. _intro-install-ubuntu: -Ubuntu 12.04 or above +Ubuntu 14.04 or above --------------------- Scrapy is currently tested with recent-enough versions of lxml, twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions. -But it should support older versions of Ubuntu too, like Ubuntu 12.04, +But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are @@ -163,8 +163,8 @@ you can install Scrapy with ``pip`` after that:: pip install scrapy .. note:: - The same non-python dependencies can be used to install Scrapy in Debian - Wheezy (7.0) and above. + The same non-Python dependencies can be used to install Scrapy in Debian + Jessue (8.0) and above. .. _intro-install-macos: diff --git a/setup.py b/setup.py index 327286f5a..2619bd544 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -61,6 +60,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', install_requires=[ 'Twisted>=13.1.0', 'w3lib>=1.17.0', diff --git a/tox.ini b/tox.ini index f35b894f3..5c543475c 100644 --- a/tox.ini +++ b/tox.ini @@ -63,25 +63,21 @@ basepython = pypy commands = py.test {posargs:scrapy tests} -[testenv:py33] -basepython = python3.3 +[testenv:py34] +basepython = python3.4 deps = -rrequirements-py3.txt # Extras Pillow -rtests/requirements-py3.txt -[testenv:py34] -basepython = python3.4 -deps = {[testenv:py33]deps} - [testenv:py35] basepython = python3.5 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} [testenv:py36] basepython = python3.6 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} [docs] changedir = docs From f11c21c6fc62b64a2bbee0e19e2098ed6257cf19 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 20 Dec 2017 17:05:56 +0200 Subject: [PATCH 083/179] Test on Python 3.4 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 66de9ed03..e2e9e0cc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy + - python: 3.4 + env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 From cbcf80b98ff66db1ccf625fa52c4de8935331972 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 20 Dec 2017 17:34:13 +0200 Subject: [PATCH 084/179] Fix typo [CI skip] --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index a2e3f506e..22bc84a40 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -164,7 +164,7 @@ you can install Scrapy with ``pip`` after that:: .. note:: The same non-Python dependencies can be used to install Scrapy in Debian - Jessue (8.0) and above. + Jessie (8.0) and above. .. _intro-install-macos: From ea41114cf0ab2782650792ad204cf43fc148c749 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 12:29:02 +0300 Subject: [PATCH 085/179] Mention PyPy support, add PyPy to install docs --- docs/faq.rst | 4 +++- docs/intro/install.rst | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 484226979..7eecc999f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,9 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.4+. +Scrapy is supported under Python 2.7 and Python 3.4+ +under CPython (default Python implementation) and PyPy (only for Python 2.7). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. +PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0. .. note:: For Python 3 support on Windows, it is recommended to use diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 22bc84a40..b00dc2cd6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,8 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.4 or above. +Scrapy runs on Python 2.7 and Python 3.4 or above +under CPython (default Python implementation) and PyPy (only for Python 2.7). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -223,6 +224,28 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy +PyPy +---- + +We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0. + +Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. +This means that these dependecies will be built during installation. +On OS X, you are likely to face an issue with building Cryptography dependency, +solution to this problem is described +`here `_, +that is to ``brew install openssl`` and then export the flags that this command +recommends (only needed when installing scrapy). Installing on Linux has no special +issues besides installing build dependencies. +Installing scrapy with PyPy on Windows is not tested. + +You can check that scrapy is installed correctly by running ``scrapy bench``. +If this command gives errors such as +``TypeError: ... got 2 unexpected keyword arguments``, this means +that setuptools was unable to pick up one PyPy-specific dependency. +To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. + + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 1058169f0e3a8646dbd20f9b4c0b599ed9f6d08e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 25 Dec 2017 15:31:07 +0500 Subject: [PATCH 086/179] setup.py: mention that we support PyPy. See GH-2213. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 2619bd544..06a36e2ba 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,8 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From f71df6f9addca10b562bb22890b5ea1c37efde5c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 13:46:22 +0300 Subject: [PATCH 087/179] Run tests for PyPy3 --- .travis.yml | 9 +++++++++ tox.ini | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index e2e9e0cc1..6635f5d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy + - python: 2.7 + env: TOXENV=pypy3 - python: 3.4 env: TOXENV=py34 - python: 3.5 @@ -30,6 +32,13 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "pypy3" ]; then + export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index 5c543475c..60ff8c15e 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,12 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:pypy3] +basepython = pypy3 +deps = {[testenv:py34]deps} +commands = + py.test {posargs:scrapy tests} + [docs] changedir = docs deps = From 041308afe7c40de7088f75b0e0c312ecd5de428a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 14:27:20 +0300 Subject: [PATCH 088/179] Fix get_func_args test for pypy3 These built-in functions are exposed as methods in PyPy3. For scrapy this does not matter as: 1) they do not work for CPython at all 2) get_func_args is checked for presense of an argument in scrapy, extra "self" does not matter. But it still makes sense to leave these tests so that we know we shouldn't use get_func_args for built-in functions/methods. --- tests/test_utils_python.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 115f523e9..f6133657b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -219,9 +219,12 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: - self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join), ['list']) - self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj']) + stripself = not six.PY2 # PyPy3 exposes them as methods + self.assertEqual( + get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, stripself), ['list']) + self.assertEqual( + get_func_args(operator.itemgetter(2), stripself), ['obj']) def test_without_none_values(self): From bb1f31189128cb2272c1302350387075fbbb730a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 15:46:05 +0300 Subject: [PATCH 089/179] Add PyPy3 support to faq and install doc --- docs/faq.rst | 4 ++-- docs/intro/install.rst | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7eecc999f..7a0628f88 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -70,10 +70,10 @@ What Python versions does Scrapy support? ----------------------------------------- Scrapy is supported under Python 2.7 and Python 3.4+ -under CPython (default Python implementation) and PyPy (only for Python 2.7). +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0. +PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. .. note:: For Python 3 support on Windows, it is recommended to use diff --git a/docs/intro/install.rst b/docs/intro/install.rst index b00dc2cd6..4a9aa3cfb 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -8,7 +8,7 @@ Installing Scrapy ================= Scrapy runs on Python 2.7 and Python 3.4 or above -under CPython (default Python implementation) and PyPy (only for Python 2.7). +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -227,7 +227,8 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0. +We recommend using the latest PyPy version. The version tested is 5.9.0. +For PyPy3, only Linux installation was tested. Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. From a1cc5a63d3e253c325159fdc6ebf4cd3faa37c49 Mon Sep 17 00:00:00 2001 From: Raphael Date: Wed, 27 Dec 2017 18:54:17 -0200 Subject: [PATCH 090/179] Add mention to dont_merge_cookies in CookiesMiddlewares docs (#2999) (#3030) Add mention to dont_merge_cookies in CookiesMiddlewares docs (#2999) --- docs/topics/downloader-middleware.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 983a93290..863620900 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -237,6 +237,17 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. +Notice that if the :class:`~scrapy.http.Request` +has ``meta['dont_merge_cookies']`` evaluated to ``True``. +despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be +sent to web servers and received cookies in +:class:`~scrapy.http.Response` will **not** be merged with the existing +cookies. + +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request` + + .. setting:: COOKIES_DEBUG COOKIES_DEBUG From 461f9daff5747728e26cd60e9dfe531092f58132 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 19:56:17 +0200 Subject: [PATCH 091/179] Update release notes for upcoming 1.4.1 version --- docs/news.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 577c93b8e..7ecf22470 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,53 @@ Release notes ============= +Scrapy 1.4.1 (2017-XX-XX) +------------------------- + +New features +~~~~~~~~~~~~ + +- Support ```` tags in ``Response.follow`` (:issue:`2785`) +- Support for ``ptpython`` REPL (:issue:`2654`) +- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`) +- Handle HTTP 308 Permanent Redirect (:issue:`2844`) +- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) +- Log versions information at startup (:issue:`2857`) +- Add template for a downloader middleware (:issue:`2755`) +- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) + +Bug fixes +~~~~~~~~~ + +- Fix PyPy test failures (:issue:`2793`) +- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) +- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) +- Add verification to check if Request callback is callable (:issue:`2766`) +- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) +- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`) +- Update deprecated test aliases (:issue:`2876`) +- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) +- Fix logging of settings overridden by ``custom_settings``; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please + update your log parsers if needed (:issue:`1343`) + +Docs +~~~~ + +- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) +- Include references to Scrapy subreddit in the docs (:issue:`2762`) +- Use https:// for readthedocs links +- Document CloseSpider extension better (:issue:`2759`) +- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) +- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`) +- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) +- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) +- Update "Releases" section in README (:issue:`2764`) +- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`) +- Small fix in description of startproject arguments (:issue:`2866`) + + Scrapy 1.4.0 (2017-05-18) ------------------------- From 45b0e1a0e4c51a773b39be14334a999cc5f0fe56 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Dec 2017 07:33:43 +0500 Subject: [PATCH 092/179] DOC draft 1.5 release notes --- docs/news.rst | 78 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7ecf22470..df0f10a32 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,51 +3,97 @@ Release notes ============= -Scrapy 1.4.1 (2017-XX-XX) +Scrapy 1.5.0 (2017-XX-XX) ------------------------- +Supported Environments +---------------------- + +* Scrapy 1.5 drops support for Python 3.3; +* this release also improves Python 3.x support (especially 3.6); +* PyPy and PyPy3 are now supported officially, by running tests on CI. +* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions. + New features ~~~~~~~~~~~~ - Support ```` tags in ``Response.follow`` (:issue:`2785`) - Support for ``ptpython`` REPL (:issue:`2654`) +- Google Cloud Storage support for FilesPipeline and ImagesPipeline + (:issue:`2923`). +- New ``--meta`` option of the "scrapy parse" command allows to pass additional + request.meta (:issue:`2883`) - Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`) - Handle HTTP 308 Permanent Redirect (:issue:`2844`) - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) - Log versions information at startup (:issue:`2857`) - Add template for a downloader middleware (:issue:`2755`) -- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) +- Explicit message for NotImplementedError when parse callback not defined + (:issue:`2831`) +- Connections to proxy servers are reused (:issue:`2743`) +- CrawlerProcess got an option to disable installation of root log handler + (:issue:`2921`) +- LinkExtractor now ignores ``m4v`` extension by default +- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) +- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and + :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`) +- Show warning when a URL is put to ``Spider.allowed_domains`` instead of + a domain (:issue:`2250`). + Bug fixes ~~~~~~~~~ -- Fix PyPy test failures (:issue:`2793`) -- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) -- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) -- Add verification to check if Request callback is callable (:issue:`2766`) -- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) -- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`) -- Update deprecated test aliases (:issue:`2876`) -- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) - Fix logging of settings overridden by ``custom_settings``; **this is technically backwards-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please update your log parsers if needed (:issue:`1343`) +- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +- Fix PyPy and PyPy3 test failures, support them officially + (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`) +- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) +- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) +- Add verification to check if Request callback is callable (:issue:`2766`) +- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) +- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning + (:issue:`2862`) +- Update deprecated test aliases (:issue:`2876`) +- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) + Docs ~~~~ -- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) -- Include references to Scrapy subreddit in the docs (:issue:`2762`) -- Use https:// for readthedocs links +- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` + setting. (:issue:`2756`) +- Update Contributing docs, document new support channels + (:issue:`2762`, issue:`3038`) +- Include references to Scrapy subreddit in the docs +- Fix broken links; use https:// for external links + (:issue:`2978`, :issue:`2982`, :issue:`2958`) - Document CloseSpider extension better (:issue:`2759`) -- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) -- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`) +- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example + (:issue:`2781`) +- Spelling mistake and typos + (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`) - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) -- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) +- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` + (:issue:`2791`) - Update "Releases" section in README (:issue:`2764`) - Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`) - Small fix in description of startproject arguments (:issue:`2866`) +- Clarify data types in Response.body docs (:issue:`2922`) +- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`) +- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware + docs (:issue:`2999`) +- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`) +- A better example of ItemExporters usage (:issue:`2989`) +- Document ``from_crawler`` methods for spider and downloader middlewares + (:issue:`3019`) + + Scrapy 1.4.0 (2017-05-18) From d4e5671d07a8dcf18b665ed3ce4136dccae222fb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Dec 2017 07:06:00 +0500 Subject: [PATCH 093/179] make release docs more readable, add highlights --- docs/news.rst | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index df0f10a32..2283b00ae 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,13 +6,35 @@ Release notes Scrapy 1.5.0 (2017-XX-XX) ------------------------- -Supported Environments ----------------------- +This release brings small new features and improvements across the codebase. +Some highlights: -* Scrapy 1.5 drops support for Python 3.3; -* this release also improves Python 3.x support (especially 3.6); -* PyPy and PyPy3 are now supported officially, by running tests on CI. -* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions. +* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline. +* Crawling with proxy servers becomes more efficient, as connections + to proxies can be reused now. +* Warnings, exception and logging messages are improved to make debugging + easier. +* ``scrapy parse`` command now allows to set custom request meta via + ``--meta`` argument. +* Compatibility with Python 3.6, PyPy and PyPy3 is improved; + PyPy and PyPy3 are now supported officially, by running tests on CI. +* Better default handling of HTTP 308, 522 and 524 status codes. +* Documentation is improved, as usual. + +Backwards Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Scrapy 1.5 drops support for Python 3.3. +* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +* Logging of settings overridden by ``custom_settings`` is fixed; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're + parsing Scrapy logs, please update your log parsers (:issue:`1343`). +* LinkExtractor now ignores ``m4v`` extension by default, this is change + in behavior. +* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`) New features ~~~~~~~~~~~~ @@ -27,20 +49,19 @@ New features - Handle HTTP 308 Permanent Redirect (:issue:`2844`) - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) - Log versions information at startup (:issue:`2857`) +- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) +- Connections to proxy servers are reused (:issue:`2743`) - Add template for a downloader middleware (:issue:`2755`) - Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) -- Connections to proxy servers are reused (:issue:`2743`) - CrawlerProcess got an option to disable installation of root log handler (:issue:`2921`) - LinkExtractor now ignores ``m4v`` extension by default -- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) - Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`) - Show warning when a URL is put to ``Spider.allowed_domains`` instead of a domain (:issue:`2250`). - Bug fixes ~~~~~~~~~ @@ -52,7 +73,8 @@ Bug fixes **This is technically backwards-incompatible**; override :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially - (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`) + (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, + :issue:`3048`) - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) - Add verification to check if Request callback is callable (:issue:`2766`) @@ -62,7 +84,6 @@ Bug fixes - Update deprecated test aliases (:issue:`2876`) - Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) - Docs ~~~~ @@ -94,8 +115,6 @@ Docs (:issue:`3019`) - - Scrapy 1.4.0 (2017-05-18) ------------------------- From c107059ef82a4b7b491b23b740b71353f03ab891 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Dec 2017 07:07:43 +0500 Subject: [PATCH 094/179] DOC fix rst syntax --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 2283b00ae..65bccc12d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -74,7 +74,7 @@ Bug fixes :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, - :issue:`3048`) + :issue:`3048`) - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) - Add verification to check if Request callback is callable (:issue:`2766`) From d07fe11981a07e493faf7454db79b98c02a53118 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 30 Dec 2017 02:09:41 +0500 Subject: [PATCH 095/179] set release date --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 65bccc12d..36ead3aba 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.5.0 (2017-XX-XX) +Scrapy 1.5.0 (2017-12-29) ------------------------- This release brings small new features and improvements across the codebase. From aa83e159c97b441167d0510064204681bbc93f21 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 30 Dec 2017 02:09:52 +0500 Subject: [PATCH 096/179] =?UTF-8?q?Bump=20version:=201.4.0=20=E2=86=92=201?= =?UTF-8?q?.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 21800f616..6e7be142e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.4.0 +current_version = 1.5.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 88c5fb891..bc80560fa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.4.0 +1.5.0 From a0836b8fd9720a9439cb3b940aca53b6844a094b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 15:59:38 +0000 Subject: [PATCH 097/179] Fix link in news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 36ead3aba..1629510b2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -98,7 +98,7 @@ Docs - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) - Spelling mistake and typos - (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`) + (:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`) - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) - Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) From 61c0b1478284b02a4fcfd2cc4931587c348c5d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 16:03:55 +0000 Subject: [PATCH 098/179] Fix typo in comment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8e9727093..038db7b47 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -379,7 +379,7 @@ class ScrapyAgent(object): {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): - # Abort connection inmediately. + # Abort connection immediately. txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) From 1d1581266c3df99ebf870d6c3e10ac09f2ee3673 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Fri, 26 Jan 2018 01:42:17 +0530 Subject: [PATCH 099/179] Changed some documentations (#3089) DOC typo fix in defer_fail docstring --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index aa6dcffda..bcf209511 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() From ba15b63ed696dbbd6aa6082c035754a15ca8e03c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jan 2018 02:11:49 +0500 Subject: [PATCH 100/179] TST fix tests to account for changes in w3lib 1.19 --- tests/test_http_response.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a36ec3af6..b228344b5 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -272,7 +272,10 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.text, u'WORD\ufffd\ufffd') + self.assertIn(r6.text, { + u'WORD\ufffd\ufffd', # w3lib < 1.19.0 + u'WORD\ufffd', # w3lib >= 1.19.0 + }) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, From c1916626c1b5c06f1b3c89cd29db1b7b5bc9996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Sat, 27 Jan 2018 21:24:15 +0000 Subject: [PATCH 101/179] Fix OS signal names --- scrapy/utils/ossignal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index df4eee5ec..f87d5a803 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,17 +1,18 @@ from __future__ import absolute_import +import signal from twisted.internet import reactor -import signal signal_names = {} for signame in dir(signal): - if signame.startswith("SIG"): + if signame.startswith('SIG') and not signame.startswith('SIG_'): signum = getattr(signal, signame) if isinstance(signum, int): signal_names[signum] = signame + def install_shutdown_handlers(function, override_sigint=True): """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the @@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True): override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows - if hasattr(signal, "SIGBREAK"): + if hasattr(signal, 'SIGBREAK'): signal.signal(signal.SIGBREAK, function) From 6f264ab190882d9bfb375688a7e44d03716242ba Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 30 Jan 2018 05:47:28 +0500 Subject: [PATCH 102/179] more stats for RobotsTxtMiddleware --- scrapy/downloadermiddlewares/robotstxt.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..b86c09c14 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,10 +41,12 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch( - to_native_str(self._useragent), request.url): + if rp is None: + return + if not rp.can_fetch(to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) + self.crawler.stats.inc_value('robotstxt/forbidden') raise IgnoreRequest() def robot_parser(self, request, spider): @@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object): dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) + self.crawler.stats.inc_value('robotstxt/request_count') if isinstance(self._parsers[netloc], Deferred): d = Deferred() @@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object): return failure def _parse_robots(self, response, netloc): + self.crawler.stats.inc_value('robotstxt/response_count') + self.crawler.stats.inc_value( + 'robotstxt/response_status_count/{}'.format(response.status)) rp = robotparser.RobotFileParser(response.url) body = '' if hasattr(response, 'text'): body = response.text - else: # last effort try + else: # last effort try try: body = response.body.decode('utf-8') except UnicodeDecodeError: @@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object): # but keep the lookup cached (in self._parsers) # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. - pass + self.crawler.stats.inc_value('robotstxt/unicode_error_count') # stdlib's robotparser expects native 'str' ; # with unicode input, non-ASCII encoded bytes decoding fails in Python2 rp.parse(to_native_str(body).splitlines()) @@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): + if failure.type is not IgnoreRequest: + key = 'robotstxt/exception_count/{}'.format(failure.type) + self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None rp_dfd.callback(None) From 4d5e5378bd2ccea2879102614536410d9338b3f1 Mon Sep 17 00:00:00 2001 From: Wenbin Zhang Date: Wed, 7 Feb 2018 10:59:32 -0500 Subject: [PATCH 103/179] Update robotstxt.py Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider --- scrapy/downloadermiddlewares/robotstxt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..4f3b5d64f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object): to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) - raise IgnoreRequest() + raise IgnoreRequest("Forbidden by robots.txt") def robot_parser(self, request, spider): url = urlparse_cached(request) From 0c374c00fb7c8d24a60bec8a94f7cbb06accb980 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 8 Feb 2018 05:09:02 +0500 Subject: [PATCH 104/179] use INFO log level to show telnet host/port --- scrapy/extensions/telnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 5ca0d19a0..e78afa1fc 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -55,9 +55,9 @@ class TelnetConsole(protocol.ServerFactory): def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() - logger.debug("Telnet console listening on %(host)s:%(port)d", - {'host': h.host, 'port': h.port}, - extra={'crawler': self.crawler}) + logger.info("Telnet console listening on %(host)s:%(port)d", + {'host': h.host, 'port': h.port}, + extra={'crawler': self.crawler}) def stop_listening(self): self.port.stopListening() From a56540877c2c24d3ba787cc43ca1f81d91b386fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 16 Jan 2018 16:14:35 -0300 Subject: [PATCH 105/179] Do not serialize unpickable objects (py3) --- scrapy/squeues.py | 7 ++++--- tests/test_squeues.py | 9 +++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 21520f454..0b8f6af7d 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python>=3.5 raises AttributeError here while - # Python<=3.4 raises pickle.PicklingError - except (pickle.PicklingError, AttributeError) as e: + # Python<=3.4 raises pickle.PicklingError here while + # Python>=3.5 raises AttributeError and + # Python>=3.6 raises TypeError + except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3a24348b4..d2f721241 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,4 @@ +from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader +from scrapy.selector import Selector class TestItem(Item): name = Field() @@ -17,20 +19,23 @@ class TestLoader(ItemLoader): name_out = staticmethod(_test_procesor) def nonserializable_object_test(self): + q = self.queue() try: pickle.dumps(lambda x: x) except Exception: # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA - q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object class A(object): pass a = A() a.__reduce__ = a.__reduce_ex__ = None - q = self.queue() self.assertRaises(ValueError, q.push, a) + if version_info.major == 3 and version_info.minor >= 6: + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From e4558cb27e8eeec8432a06124acf8c2569784ccc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 19 Jan 2018 10:51:30 -0300 Subject: [PATCH 106/179] Update test for unpickable objects --- tests/test_squeues.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2f721241..3ded5c027 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,4 +1,3 @@ -from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -32,10 +31,9 @@ def nonserializable_object_test(self): a = A() a.__reduce__ = a.__reduce_ex__ = None self.assertRaises(ValueError, q.push, a) - if version_info.major == 3 and version_info.minor >= 6: - # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) - sel = Selector(text='

some text

') - self.assertRaises(ValueError, q.push, sel) + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From 0d87e77afeb506c69f4717744917b95234a86650 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 14:49:26 -0300 Subject: [PATCH 107/179] Bump parsel dependency --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 392f83dd6..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=1.1 +parsel>=1.4 diff --git a/setup.py b/setup.py index 06a36e2ba..c37919cda 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.1', + 'parsel>=1.4', 'PyDispatcher>=2.0.5', 'service_identity', ], From 6edd4114c4e715a3a0c440af455fff089a099620 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 15:47:20 -0300 Subject: [PATCH 108/179] Clarify comment about Pyhton versions --- scrapy/squeues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 0b8f6af7d..d2074a457 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python<=3.4 raises pickle.PicklingError here while - # Python>=3.5 raises AttributeError and - # Python>=3.6 raises TypeError + # Python <= 3.4 raises pickle.PicklingError here while + # 3.5 <= Python < 3.6 raises AttributeError and + # Python >= 3.6 raises TypeError except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) From dc0304fde1b29b4973fabf9d189eb5c4084bf899 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 13 Feb 2018 19:47:41 +0500 Subject: [PATCH 109/179] fix docs building with recent sphinx: don't use deprecated sphinx options and imports --- docs/_ext/scrapydocs.py | 6 +++++- docs/conf.py | 4 ---- docs/requirements.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 83b0d2cc6..192123473 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,6 +1,6 @@ from docutils.parsers.rst.roles import set_classes from docutils import nodes -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode from operator import itemgetter @@ -110,24 +110,28 @@ def setup(app): app.connect('doctree-read', collect_scrapy_settings_refs) app.connect('doctree-resolved', replace_settingslist_nodes) + def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/issues/' + text set_classes(options) node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) return [node], [] + def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/commit/' + text set_classes(options) node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) return [node], [] + def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'http://hg.scrapy.org/scrapy/changeset/' + text set_classes(options) diff --git a/docs/conf.py b/docs/conf.py index 007dc2788..594740f39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,10 +144,6 @@ html_static_path = ['_static'] # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -html_use_smartypants = True - # Custom sidebar templates, maps document names to template names. #html_sidebars = {} diff --git a/docs/requirements.txt b/docs/requirements.txt index d3dcb97be..8e7611d21 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.3 +Sphinx>=1.6 sphinx_rtd_theme \ No newline at end of file From 6954da136604ef2a5bf37f8d792e0de6ec90469a Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Thu, 15 Feb 2018 23:27:40 +0530 Subject: [PATCH 110/179] Updated contributing.rst Rectified grammatical errors --- docs/contributing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9a02634cb..44068baa9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,7 +6,7 @@ Contributing to Scrapy .. important:: - Double check you are reading the most recent version of this document at + Double check that you are reading the most recent version of this document at https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: @@ -18,7 +18,7 @@ There are many ways to contribute to Scrapy. Here are some of them: * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. -* Submit patches for new functionality and/or bug fixes. Please read +* Submit patches for new functionalities and/or bug fixes. Please read :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. @@ -80,8 +80,8 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, the higher chance that it'll get accepted and -the sooner that will be merged. +The better written a patch is, higher is the chance that it'll get accepted and +sooner it will be merged. Well-written patches should: From bbc2a3569f153b20bf306375883688d99b565cb3 Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Fri, 16 Feb 2018 23:33:10 +0530 Subject: [PATCH 111/179] further edited --- docs/contributing.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 44068baa9..f4f9e393f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -80,8 +80,7 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, higher is the chance that it'll get accepted and -sooner it will be merged. +The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: From acd2b8d43b5ebec7ffd364b6f335427041a0b98d Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Thu, 22 Feb 2018 06:37:26 +0800 Subject: [PATCH 112/179] [MRG+1] Fix part of issue #3128 - None should not be a valid type for 'url' in Response.follow (#3131) * fix one issue of issue#3128 because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.' * fix one issue of issue#3128 because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.' --- scrapy/http/response/__init__.py | 2 ++ tests/test_http_response.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 434d87eab..1974259b5 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -120,6 +120,8 @@ class Response(object_ref): """ if isinstance(url, Link): url = url.url + elif url is None: + raise ValueError("url can't be None") url = self.urljoin(url) return Request(url, callback, method=method, diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b228344b5..820758dc9 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -155,6 +155,10 @@ class BaseResponseTest(unittest.TestCase): self._assert_followed_url(Link('http://example.com/foo'), 'http://example.com/foo') + def test_follow_None_url(self): + r = self.response_class("http://example.com") + self.assertRaises(ValueError, r.follow, None) + def test_follow_whitespace_url(self): self._assert_followed_url('foo ', 'http://example.com/foo%20') From aca2655c12d806759c6e0821a40d0277d200e0ea Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 14:57:39 +0100 Subject: [PATCH 113/179] [WIP] Run tests for Python 3.7 --- .travis.yml | 2 ++ tox.ini | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..e4df22139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs + - python: 3.7 + env: TOXENV=py37 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/tox.ini b/tox.ini index 60ff8c15e..5301624ee 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,10 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:py37] +basepython = python3.7 +deps = {[testenv:py34]deps} + [testenv:pypy3] basepython = pypy3 deps = {[testenv:py34]deps} From fab68ff6260b9ce4f55ca7b211a1aeb3e8e6df3d Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Fri, 2 Mar 2018 17:05:14 +0100 Subject: [PATCH 114/179] Use 3.7-dev version for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e4df22139..aa1a3c4c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7 + - python: 3.7-dev env: TOXENV=py37 install: - | From 4c05441450bc1f8438239af0310ab76777a2dacf Mon Sep 17 00:00:00 2001 From: nctl144 Date: Sat, 3 Mar 2018 00:00:03 -0500 Subject: [PATCH 115/179] add ftp to the scheme list --- scrapy/linkextractors/__init__.py | 3 ++- tests/test_linkextractors.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..cda6ddc7e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ + 'file', 'ftp'} class FilteringLinkExtractor(object): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1d7c4f311..903032b52 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -451,6 +451,17 @@ class Base: Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_ftp_links(self): + body = b""" + + + """ + response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + ]) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor @@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() - From d5b7ebcfdcfd40d29990712b587893fcc6e84ce8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 18:17:49 +0530 Subject: [PATCH 116/179] Fixed bug FormRequest.from_response() clickdata ignores input[type=image] --- scrapy/http/request/form.py | 10 ++++++---- tests/test_http_request.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9d178a3e..184ee2599 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,10 +170,12 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: return diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fca8ff411..73a74cd5d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,6 +532,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( From 65744c2199fc6a5bccfa11eec40a867c1401aee9 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 20:07:50 +0530 Subject: [PATCH 117/179] Corrected Test --- tests/test_http_request.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 73a74cd5d..a042f03b6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,16 +532,6 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""") - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -554,6 +544,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( From ca7d79c29a55be4482bf2d4f704fc145ba801337 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 10:46:51 +0100 Subject: [PATCH 118/179] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..47eddf1fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +git+https://github.com/twisted/twisted.git@dcaf946 lxml pyOpenSSL cssselect>=0.9 From 5d1f5245f2699745e73449b013a29cc370f424c9 Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:14:50 +0100 Subject: [PATCH 119/179] [WIP] Install Twisted from branch to bypass syntax issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47eddf1fc..95cd37772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/twisted/twisted.git@dcaf946 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml pyOpenSSL cssselect>=0.9 From f10a43d562dee32f324703fddb19bee5266912ce Mon Sep 17 00:00:00 2001 From: Patience Shyu Date: Mon, 5 Mar 2018 11:43:39 +0100 Subject: [PATCH 120/179] [WIP] Install Twisted from branch for py3.7 --- requirements-py3.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..c3357e970 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +git+https://github.com/lopuhin/twisted.git@9384-remove-async-param lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/requirements.txt b/requirements.txt index 95cd37772..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 From 13a74d77e2e4d1719bf984d99ca2ae6874752e41 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 12 Mar 2018 22:25:19 +0800 Subject: [PATCH 121/179] catch CertificateError in tls verification --- scrapy/core/downloader/tls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 498e3d60f..e1c4f4908 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,6 +40,7 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) + from service_identity.exceptions import CertificateError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -65,7 +66,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except VerificationError as e: + except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) From e487100987496474de4d2b5fc509ff47d01cd0b6 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 08:59:03 +0800 Subject: [PATCH 122/179] add a test case --- tests/keys/localhost-ip.gen.README | 21 ++++++++++++ tests/keys/localhost.crt | 36 ++++++++++----------- tests/keys/localhost.ip.crt | 20 ++++++++++++ tests/keys/localhost.ip.key | 28 ++++++++++++++++ tests/keys/localhost.key | 52 +++++++++++++++--------------- tests/test_downloader_handlers.py | 9 ++++++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 tests/keys/localhost-ip.gen.README create mode 100644 tests/keys/localhost.ip.crt create mode 100644 tests/keys/localhost.ip.key diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README new file mode 100644 index 000000000..8e94e1217 --- /dev/null +++ b/tests/keys/localhost-ip.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 +Email Address []:. + diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 13c5b5bd6..48d7bd9a3 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x -NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K -o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y -njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY -ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ -8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 -gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O -BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j -xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES -9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i -jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ -TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own -tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM -am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U -Mjybr/hQK3mnVdE= +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt new file mode 100644 index 000000000..48d7bd9a3 --- /dev/null +++ b/tests/keys/localhost.ip.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key new file mode 100644 index 000000000..1e12c1255 --- /dev/null +++ b/tests/keys/localhost.ip.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq +-----END PRIVATE KEY----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index da975e6d3..1e12c1255 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 -ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 -JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz -vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 -RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY -b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ -XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 -gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk -i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu -Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss -ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 -hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD -7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa -EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf -uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG -HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR -UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO -j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri -fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ -6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI -xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g -DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f -npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM -8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 -J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj -Z+pMCFLbwAbFZ9Qb/NFlUv0= +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq -----END PRIVATE KEY----- diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bd2c86292..ceb03f945 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -505,6 +505,15 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' +class Https11InvalidDNSPattern(Https11TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + super(Https11InvalidDNSPattern, self).setUp() + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" From d9e6c73fb3ef787e39474bda5f008b309b65c65b Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 13:05:37 +0800 Subject: [PATCH 123/179] revert wrong changes --- tests/keys/localhost.crt | 36 ++++++++++++++-------------- tests/keys/localhost.key | 52 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 48d7bd9a3..13c5b5bd6 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x -ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL -JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev -y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV -ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR -v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 -JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O -BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ -dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr -idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP -8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X -kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ -FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb -smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO -zh6AcsnjkLumOkM= +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index 1e12c1255..da975e6d3 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo -+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC -T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb -rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR -3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi -JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj -uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES -FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc -BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud -JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD -bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT -ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx -KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N -yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA -hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ -32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie -PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY -r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 -X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz -ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ -Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ -g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 -WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ -inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u -BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz -m1Q+9xJdtRxlYc1YGTK1YGCq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= -----END PRIVATE KEY----- From 6a7cdf9a6c4162cf4dd91721d28974c80e0f68cd Mon Sep 17 00:00:00 2001 From: siulkilulki Date: Tue, 13 Mar 2018 08:35:27 +0100 Subject: [PATCH 124/179] [MRG+1] Add 'flv' to ignored video extensions. (#3165) --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..c3c79cf25 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', 'm4v', + 'm4a', 'm4v', 'flv', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From 1a2f0193a30c1aceb4743e33a90b3c264e0b09c4 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 19:14:52 +0800 Subject: [PATCH 125/179] fix tests on jessie --- scrapy/core/downloader/tls.py | 11 +++++++++-- tests/test_downloader_handlers.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e1c4f4908..c97c6a9a9 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,7 +40,14 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) - from service_identity.exceptions import CertificateError + try: + # XXX: this import would fail on Debian jessie with system installed + # service_identity library, due to lack of cryptography.x509 dependency + # See https://github.com/pyca/service_identity/issues/21 + from service_identity.exceptions import CertificateError + verification_errors = (CertificateError, VerificationError) + except ImportError: + verification_errors = VerificationError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -66,7 +73,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except (CertificateError, VerificationError) as e: + except verification_errors as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ceb03f945..b34faa7e7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -512,6 +512,10 @@ class Https11InvalidDNSPattern(Https11TestCase): certfile = 'keys/localhost.ip.crt' def setUp(self): + try: + from service_identity.exceptions import CertificateError + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") super(Https11InvalidDNSPattern, self).setUp() From 2c58da19a6f85f4532d80d3a941cede8f9d0bab8 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 14 Mar 2018 09:27:59 +0800 Subject: [PATCH 126/179] update docstring of ScrapyClientTLSOptions --- scrapy/core/downloader/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index c97c6a9a9..df8051182 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -63,8 +63,9 @@ if twisted_version >= (14, 0, 0): (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError and ValueError exceptions are caught, - so that the connection is not closed, only logging warnings. + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): From ff5f717f7a2aaf0a1a1101019485201427edd536 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:17:48 +0530 Subject: [PATCH 127/179] Fixed formatting issues --- scrapy/http/request/form.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 184ee2599..22846ad77 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,15 +168,17 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ + print("form =", form.__dict__) clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] + print("clickables =", clickables) if not clickables: return From e25e2afe174bbe70ecbf88ea684937890e8ba4d5 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:20:14 +0530 Subject: [PATCH 128/179] Removed unnecessary print statements --- scrapy/http/request/form.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 22846ad77..238dd44b3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,7 +168,6 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - print("form =", form.__dict__) clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' @@ -178,7 +177,6 @@ def _get_clickable(clickdata, form): '|descendant::button[not(@type)]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] - print("clickables =", clickables) if not clickables: return From a5acc9373f8735e27c22de6fbe345fbed8f268c1 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 18:19:39 +0530 Subject: [PATCH 129/179] Resolving Comments --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 238dd44b3..d033a830e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,11 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', + 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From dd064413a46356940151ac3f9ccd8a45bca2cbd8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 19:28:41 +0530 Subject: [PATCH 130/179] corrected syntax error in XPath --- scrapy/http/request/form.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d033a830e..95b38e990 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,8 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From c6d20bdd826070c6c808421cd6331b5d208f8aa7 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 27 Mar 2018 16:21:07 -0400 Subject: [PATCH 131/179] Doc: update wording for COOKIES_ENABLED --- docs/topics/downloader-middleware.rst | 20 +++++++++----------- docs/topics/request-response.rst | 4 +++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 863620900..dfe4c13b4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -158,13 +158,13 @@ more of the following methods: :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) - + If present, this classmethod is called to create a middleware instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the middleware. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for middleware to access them and hook its functionality into Scrapy. - + :param crawler: crawler that uses this middleware :type crawler: :class:`~scrapy.crawler.Crawler` object @@ -237,16 +237,14 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. -Notice that if the :class:`~scrapy.http.Request` -has ``meta['dont_merge_cookies']`` evaluated to ``True``. -despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be -sent to web servers and received cookies in -:class:`~scrapy.http.Response` will **not** be merged with the existing -cookies. - -For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request` +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. .. setting:: COOKIES_DEBUG diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 121abe6b5..e29914dbf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -80,6 +80,8 @@ Request objects attributes of the cookie. This is only useful if the cookies are saved for later requests. + .. reqmeta:: dont_merge_cookies + When some site returns cookies (in a response) those are stored in the cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. However, if, for some @@ -294,7 +296,7 @@ Those are: * :reqmeta:`dont_retry` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`handle_httpstatus_all` -* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor) +* :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` * :reqmeta:`redirect_urls` From 8e8994c6b55fa8e975ce30b25ae326f829a58aed Mon Sep 17 00:00:00 2001 From: rhoboro Date: Mon, 2 Apr 2018 15:36:47 +0900 Subject: [PATCH 132/179] add acl support for gcs --- scrapy/pipelines/files.py | 6 +++++- scrapy/pipelines/images.py | 1 + scrapy/settings/default_settings.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ab18a727d..af1d5488a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,6 +207,8 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' + POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in + # FilesPipeline.from_settings. def __init__(self, uri): from google.cloud import storage @@ -239,7 +241,8 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type=self._get_content_type(headers) + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY ) @@ -314,6 +317,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c5fc12afe..5cdddce49 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ead511473..7916b9704 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = 'projectPrivate' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = 'projectPrivate' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 5254ac393bdf712db698bfa41caf5fb2e682f883 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:00:08 +0900 Subject: [PATCH 133/179] added test for gcs policy --- scrapy/utils/test.py | 3 ++- tests/test_pipeline_files.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 60b931f48..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path): bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted bucket.delete_blob(path) - return content, blob + return content, acl, blob def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c761bd606..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase): meta = {'foo': 'bar'} path = 'full/filename' store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) self.assertIn('last_modified', s) self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) class ItemWithFiles(Item): From 74a9c65290888b3ce712c3cfa7132f91ffa3c576 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:20:37 +0900 Subject: [PATCH 134/179] update docs for support gcs acl --- docs/topics/media-pipeline.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 41beebe98..284ec1a25 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -189,6 +189,8 @@ Google Cloud Storage --------------------- .. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) @@ -204,6 +206,18 @@ For information about authentication, see this `documentation`_. .. _documentation: https://cloud.google.com/docs/authentication/production +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``projectPrivate``. To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + Usage example ============= From cb76b88331e1e0cff30de9a6961de3e28e94ff44 Mon Sep 17 00:00:00 2001 From: grammy-jiang Date: Wed, 4 Apr 2018 05:56:05 -0400 Subject: [PATCH 135/179] fix a mistake in topic spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..1d451af21 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -116,7 +116,7 @@ following methods: method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, From 464973489e9f457570d7dd1a2e4aab8c4c1778fd Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 12:06:39 +0900 Subject: [PATCH 136/179] Using bucket's default object ACL --- docs/topics/media-pipeline.rst | 3 ++- scrapy/pipelines/files.py | 6 ++++-- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 284ec1a25..0872ac0cd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,8 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -``projectPrivate``. To make the files publicly available use the ``publicRead`` +None which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` policy:: IMAGES_STORE_GCS_ACL = 'publicRead' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index af1d5488a..8ea70e5d1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,8 +207,10 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' - POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in - # FilesPipeline.from_settings. + + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None def __init__(self, uri): from google.cloud import storage diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7916b9704..d7ca8a835 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = 'projectPrivate' +FILES_STORE_GCS_ACL = None FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = 'projectPrivate' +IMAGES_STORE_GCS_ACL = None ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 560ee623fd84c3db986fccacaef7e9a31a8ea02c Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:00:27 +0900 Subject: [PATCH 137/179] set defalut value "" to FILES_STORE_GCS_ACL --- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8ea70e5d1..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -319,7 +319,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5cdddce49..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,7 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7ca8a835..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = None +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = None +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 6ef6585b5a187d4a8dcec99ba7cae9b6cee91b30 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:06:29 +0900 Subject: [PATCH 138/179] update docs --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0872ac0cd..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,7 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -None which means that Cloud Storage applies the bucket's default object ACL to the object. +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. To make the files publicly available use the ``publicRead`` policy:: From 57b0e6b6955705efcbe2d3da5501b27716ef1baf Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Thu, 19 Apr 2018 13:35:46 +0800 Subject: [PATCH 139/179] improve document about functions as processors --- docs/topics/loaders.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index ad86dba63..cdb066e57 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. +If you want to use a plain function as a processor, make sure it receives +``self`` as the first argument:: + + def lowercase_processor(self, values): + for v in values: + yield v.lower() + + class MyItemLoader(ItemLoader): + name_in = lowercase_processor + +This is because whenever a function is assigned as a class variable, it becomes +a method and would be passed the instance as the the first argument when being +called. See `this answer on stackoverflow`_ for more details. + The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -143,6 +157,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== From e75f721c04446f8f28d3bdfcd69f967f65981407 Mon Sep 17 00:00:00 2001 From: Pengyu Chen Date: Mon, 23 Apr 2018 22:08:28 +0800 Subject: [PATCH 140/179] Added: Allowing optional arguments for `scrapy.http.cookies.CookieJar.clear` --- scrapy/http/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a1e95102e..4e8056750 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -58,8 +58,8 @@ class CookieJar(object): def clear_session_cookies(self, *args, **kwargs): return self.jar.clear_session_cookies(*args, **kwargs) - def clear(self): - return self.jar.clear() + def clear(self, domain=None, path=None, name=None): + return self.jar.clear(domain, path, name) def __iter__(self): return iter(self.jar) From 0d015e5c0f0dcb8936044aec163dd12e33482730 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 16 May 2018 09:36:07 +0000 Subject: [PATCH 141/179] blacklist twisted version with regression --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..7d857a8c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +Twisted>=13.1.0,!=18.4.0 lxml pyOpenSSL cssselect>=0.9 From c5ddfddb7e5b42e45b6c8f86a381ee52fbb6f797 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 17 May 2018 08:53:42 +0000 Subject: [PATCH 142/179] blacklist twisted version with regression in constraints file --- requirements.txt | 2 +- tests/constraints.txt | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/constraints.txt diff --git a/requirements.txt b/requirements.txt index 7d857a8c9..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0,!=18.4.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tox.ini b/tox.ini index 60ff8c15e..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore From 2dfc5d128bba42e2fe2bb24c0326fe47b0d3cd97 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 11:59:38 -0400 Subject: [PATCH 143/179] Update DEPTH_STATS refs to DEPTH_STATS_VERBOSE --- docs/topics/settings.rst | 11 ----------- docs/topics/spider-middleware.rst | 3 ++- scrapy/settings/default_settings.py | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 076dc6bfd..1f1217770 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -335,17 +335,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` and :setting:`RETRY_PRIORITY_ADJUST`. -.. setting:: DEPTH_STATS - -DEPTH_STATS ------------ - -Default: ``True`` - -Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` - -Whether to collect maximum depth stats. - .. setting:: DEPTH_STATS_VERBOSE DEPTH_STATS_VERBOSE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..265acdb43 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -212,7 +212,8 @@ DepthMiddleware * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to crawl for any site. If zero, no limit will be imposed. - * :setting:`DEPTH_STATS` - Whether to collect depth stats. + * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of + requests for each depth. * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on their depth. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 36e17ef6b..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -55,7 +55,7 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS = True +DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 DNSCACHE_ENABLED = True From 6a182c955273745daf334033944c82da3aa4eb12 Mon Sep 17 00:00:00 2001 From: Ryan P Kilby Date: Wed, 9 May 2018 12:00:18 -0400 Subject: [PATCH 144/179] Depth stats are not optional --- scrapy/spidermiddlewares/depth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index e2f039146..34a87f2df 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class DepthMiddleware(object): - def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1): + def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth self.stats = stats self.verbose_stats = verbose_stats @@ -41,7 +41,7 @@ class DepthMiddleware(object): extra={'spider': spider} ) return False - elif self.stats: + else: if self.verbose_stats: self.stats.inc_value('request_depth_count/%s' % depth, spider=spider) @@ -50,7 +50,7 @@ class DepthMiddleware(object): return True # base case (depth=0) - if self.stats and 'depth' not in response.meta: + if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) From b364d27247b2d9b86c164569c7e0459fa3f8391b Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 23 May 2018 21:25:50 +0300 Subject: [PATCH 145/179] [MRG+1] Automatic port selection for servicies in unit tests (#3210) * ability to pass port as a parameter * try to find free ports * use environment variables to pass mock server address * get mock server address from environment variables * ability to select ports for proxy in runtime * use common method for URLs from mock server * https support * get mock server address * get mock address * replace hand-written mechanism by kernel-based one * use ephemeral ports in mockserver * strip EOL from addresses * use ephemeral port in proxy * no need to restore environment as it is restored in tearDown * decode bytes * use mockserver address as a variable * ability to pass address as variable * per test-case mockserver * use base class * remove obsolete environment manipulation * return usage of proxy for http cases * common method for broking proxy auth credentials * python version-independent url methods --- tests/mockserver.py | 25 +++++++++--- tests/spiders.py | 17 ++++++--- tests/test_closespider.py | 8 ++-- tests/test_crawl.py | 48 ++++++++++++------------ tests/test_downloader_handlers.py | 8 ++-- tests/test_feedexport.py | 3 +- tests/test_pipeline_crawl.py | 16 ++++---- tests/test_proxy_connect.py | 41 ++++++++++++-------- tests/test_spidermiddleware_httperror.py | 23 ++++++------ 9 files changed, 111 insertions(+), 78 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 98723846e..f36ce3c44 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -192,9 +192,15 @@ class MockServer(): def __enter__(self): from scrapy.utils.test import get_testenv + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) - self.proc.stdout.readline() + http_address = self.proc.stdout.readline().strip().decode('ascii') + https_address = self.proc.stdout.readline().strip().decode('ascii') + + self.http_address = http_address + self.https_address = https_address + return self def __exit__(self, exc_type, exc_value, traceback): @@ -202,6 +208,12 @@ class MockServer(): self.proc.wait() time.sleep(0.2) + def url(self, path, is_secure=False): + host = self.http_address + if is_secure: + host = self.https_address + return host + path + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'): return ssl.DefaultOpenSSLContextFactory( @@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, factory) + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(8999, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - print("Mock server running at http://%s:%d and https://%s:%d" % ( - httpHost.host, httpHost.port, httpsHost.host, httpsHost.port)) + httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) + httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/spiders.py b/tests/spiders.py index 1038b69de..7816bf7c7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -11,7 +11,12 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor -class MetaSpider(Spider): +class MockServerSpider(Spider): + def __init__(self, mockserver=None, *args, **kwargs): + super(MockServerSpider, self).__init__(*args, **kwargs) + self.mockserver = mockserver + +class MetaSpider(MockServerSpider): name = 'meta' @@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) self.start_urls = [url] def parse(self, response): @@ -55,7 +60,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b) + url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider): return self.errback_func(failure) -class DuplicateStartRequestsSpider(Spider): +class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True name = 'duplicatestartrequests' distinct_urls = 2 @@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = "http://localhost:8998/echo?headers=1&body=test%d" % i + url = self.mockserver.url("/echo?headers=1&body=test%d" % i) yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index fa0b48998..0eb1b7944 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -18,7 +18,7 @@ class TestCloseSpider(TestCase): def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') itemcount = crawler.stats.get_value('item_scraped_count') @@ -28,7 +28,7 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') pagecount = crawler.stats.get_value('response_received_count') @@ -38,7 +38,7 @@ class TestCloseSpider(TestCase): def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ @@ -50,7 +50,7 @@ class TestCloseSpider(TestCase): def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') stats = crawler.stats diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3fc13eeb7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -26,7 +26,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks @@ -42,7 +42,7 @@ class CrawlTestCase(TestCase): def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2) + yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) @@ -53,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): crawler = self.runner.create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @@ -61,13 +61,13 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -77,14 +77,14 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/status?n=503") + yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:65432/status?n=503") + yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -92,14 +92,14 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./") + yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1) + yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1) + yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -121,7 +121,7 @@ class CrawlTestCase(TestCase): def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), # crawler.spider.seedsseen) @@ -130,10 +130,10 @@ class CrawlTestCase(TestCase): def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -160,7 +160,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) + yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) self.assertEqual(str(l).count("Got response 200"), 1) @defer.inlineCallbacks @@ -168,7 +168,7 @@ with multiples lines # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=0") + yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -176,7 +176,7 @@ with multiples lines # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=1") + yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(l) def _assert_retried(self, log): @@ -186,7 +186,7 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1) + req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) req1 = req0.replace() req2 = req0.replace(headers={'Referer': None}) req3 = req0.replace(headers={'Referer': 'http://example.com'}) @@ -194,7 +194,7 @@ with multiples lines req1.meta['next'] = req2 req2.meta['next'] = req3 crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0) + yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) @@ -220,7 +220,7 @@ with multiples lines est.append(get_engine_status(crawler.engine)) crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) self.assertEqual(s['engine.spider.name'], crawler.spider.name) @@ -244,7 +244,7 @@ with multiples lines raise TestError crawler = self.runner.create_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(), TestError) + yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -256,7 +256,7 @@ with multiples lines } crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) yield self.assertFailure( - self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @@ -264,13 +264,13 @@ with multiples lines def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: yield self.runner.join() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b34faa7e7..c91be2c0c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000})) + yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) failure = crawler.spider.meta['failure'] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=Request(url='http://localhost:8998')) + yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] @@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase): def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider) body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', + request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] @@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase): if six.PY2: request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') + request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f55927121..0d9f1e83c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase): try: with MockServer() as s: runner = CrawlerRunner(Settings(defaults)) + spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) with open(res_name, 'rb') as f: @@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): for item in items: @@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): pass diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9b81f827d..5985a6f3e 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): def _process_url(self, url): return add_or_replace_parameter( - 'http://localhost:8998/redirect-to', + self.mockserver.url('/redirect-to'), 'goto', url) @@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6213a51e8..ae1236bcb 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import time +from six.moves.urllib.parse import urlsplit, urlunsplit from threading import Thread from libmproxy import controller, proxy from netlib import http_auth @@ -17,7 +18,7 @@ from tests.mockserver import MockServer class HTTPSProxy(controller.Master, Thread): - def __init__(self, port): + def __init__(self): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread): server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, cacert = cert_path), - port) + 0) + self.server = server Thread.__init__(self) controller.Master.__init__(self, server) + def http_address(self): + return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): @@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy(8888) + + self._proxy = HTTPSProxy() self._proxy.start() + # Wait for the proxy to start. time.sleep(1.0) - os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = self._proxy.http_address() + os.environ['http_proxy'] = self._proxy.http_address() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase): def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) @defer.inlineCallbacks def test_https_noconnect(self): - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error(l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request("https://localhost:8999/echo") + request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) with LogCapture() as l: yield crawler.crawl(seed=request) @@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 19e6bbdcd..dacd0147f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,20 +11,21 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.spiders import MockServerSpider -class _HttpErrorSpider(Spider): +class _HttpErrorSpider(MockServerSpider): name = 'httperror' - start_urls = [ - "http://localhost:8998/status?n=200", - "http://localhost:8998/status?n=404", - "http://localhost:8998/status?n=402", - "http://localhost:8998/status?n=500", - ] bypass_status_codes = set() def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) + self.start_urls = [ + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), + ] self.failed = set() self.skipped = set() self.parsed = set() @@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: - yield crawler.crawl(bypass_status_codes={402}) + yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) self.assertEqual(crawler.spider.parsed, {'200', '402'}) self.assertEqual(crawler.spider.skipped, {'402'}) self.assertEqual(crawler.spider.failed, {'404', '500'}) @@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) From ffa7bede17088e5eebd50305704906bf1451ab3a Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Wed, 30 May 2018 06:33:18 +0200 Subject: [PATCH 146/179] Update spiders.rst I changed URLs to :class:`~scrapy.http.Request` in start_urls explanation of the default spider --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c2c271245..697732b47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -88,7 +88,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data + listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings From ecdd888ff4614a0e994c9dadb31b7c9c85c88e8a Mon Sep 17 00:00:00 2001 From: Chris Slothouber Date: Fri, 1 Jun 2018 09:25:34 -0400 Subject: [PATCH 147/179] Minor edits to contributing.rst Corrected minor grammatical issues and increased clarity of instructions. --- docs/contributing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f4f9e393f..6615840f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -48,9 +48,9 @@ guidelines when reporting a new bug. `Stack Overflow `__ (use "scrapy" tag). -* check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report, but check the ticket history and comments. If you - have additional useful information, please leave a comment, or consider +* check the `open issues`_ to see if the issue has already been reported. If it + has, don't dismiss the report, but check the ticket history and comments. If + you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has @@ -122,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first. Sometimes there is an existing pull request for the problem you'd like to solve, which is stalled for some reason. Often the pull request is in a right direction, but changes are requested by Scrapy maintainers, and the -original pull request author haven't had time to address them. +original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is @@ -143,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports -removal, etc) in separate commits than functional changes. This will make pull +removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. Coding style @@ -170,7 +170,7 @@ Documentation policies **do** provide a docstring, but make sure sphinx documentation uses autodoc_ extension to pull the docstring. For example, the :meth:`ItemLoader.add_value` method should be either - documented only in the sphinx documentation (not it a docstring), or + documented only in the sphinx documentation (not as a docstring), or it should have a docstring which is pulled to sphinx documentation using autodoc_ extension. From 6a2d2c3b77bde1d74b46d7dbfb9488cb06e5021f Mon Sep 17 00:00:00 2001 From: Fredrik Bergenlid Date: Fri, 1 Jun 2018 21:38:07 +0200 Subject: [PATCH 148/179] Improve gunzip performance for big files --- scrapy/utils/gz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 16c9ce539..ec3949651 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -30,25 +30,25 @@ def gunzip(data): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output = b'' + output_list = [] chunk = b'.' while chunk: try: chunk = read1(f, 8196) - output += chunk + output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output is '' and f.extrabuf + # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output or getattr(f, 'extrabuf', None): + if output_list or getattr(f, 'extrabuf', None): try: - output += f.extrabuf[-f.extrasize:] + output_list.append(f.extrabuf[-f.extrasize:]) finally: break else: raise - return output + return b''.join(output_list) _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search From 98d9093dc7241b32b927e79ff2f8a475ae5659a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:37:16 -0700 Subject: [PATCH 149/179] minor grammatical fixes in CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 162602248..d477168eb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -3,7 +3,7 @@ ## Our Pledge In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and +contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and From 9bd5444a42e82a05d2791fd10a205a9dd99303a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:48:28 -0700 Subject: [PATCH 150/179] added oxford commas to LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 6ead05ece..4d0a0863a 100644 --- a/LICENSE +++ b/LICENSE @@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. + this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the + notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of Scrapy nor the names of its contributors may be used From 12d10eec2cb10fb13cf199d73363093478e80f9a Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:53:53 -0700 Subject: [PATCH 151/179] changed Twisted >= 17.9.0 to Twisted>=17.9.0 --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..1f342cfbb 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From 596f39600dfa94504b585ef65b1b04571a631447 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Sat, 19 May 2018 16:32:55 -0700 Subject: [PATCH 152/179] reversed earlier change that seemed to cause Travis CI build failure --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 1f342cfbb..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted>=17.9.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From d4511667fb5df63058accfe731e3d4160e795ee8 Mon Sep 17 00:00:00 2001 From: mugayoshi Date: Sat, 9 Jun 2018 18:17:11 +0900 Subject: [PATCH 153/179] Update debugging memory leaks section in the docs Add Python3 tools description. --- docs/topics/leaks.rst | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 92590c180..af14d14e8 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. +If you're using Python3, see :ref:`topics-leaks-muppy`. .. _Guppy library: https://pypi.python.org/pypi/guppy @@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the .. _Guppy documentation: http://guppy-pe.sourceforge.net/ +.. _topics-leaks-muppy: + +Debugging memory leaks with muppy +================================= +If you're using Python 3, you can use muppy from `Pympler`_. + +.. _Pympler: https://pypi.org/project/Pympler/ + +If you use ``pip``, you can install muppy with the following command:: + + pip install Pympler + +Here's an example to view all Python objects available in +the heap using muppy:: + + >>> from pympler import muppy + >>> all_objects = muppy.get_objects() + >>> len(all_objects) + 28667 + >>> from pympler import summary + >>> suml = summary.summarize(all_objects) + >>> summary.print_(suml) + types | # objects | total size + ==================================== | =========== | ============ + Date: Wed, 13 Jun 2018 18:11:43 -0300 Subject: [PATCH 154/179] Include Python version indication to each required library used in S3 storage --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 135d05c93..b64dbfbfd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ or `boto`_ + * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only) The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: From 72d0899bce06190de5a453b24dd66c8910e6d0ee Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 14 Jun 2018 17:58:48 +0300 Subject: [PATCH 155/179] Return non-zero exit code from scrapy commands in case of spider bootstrap errors * method to detect spider creation in crawler * correct method name * method to know if crawlers has spiders * we do not need to issue requests * set exit code accordingly to spiders in crawlers * more portable way to check ofr exceptions * more clear way * test cases for several spiders per crawler * grammatically correct name for method * method is private * grammatically correct name for method * method is private * remove unused import * correct order of imports * changes mechanism of obtaining spider status from method to object member * rename tests --- scrapy/commands/crawl.py | 3 ++ scrapy/commands/runspider.py | 3 ++ scrapy/crawler.py | 2 ++ tests/test_commands.py | 12 +++++++ tests/test_crawler.py | 64 +++++++++++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b986bf9d..8093fd402 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -56,3 +56,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a98033dd1..376d3c84e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -87,3 +87,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5cbc2d7c5..04aee18ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -137,6 +137,7 @@ class CrawlerRunner(object): self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() + self.bootstrap_failed = False @property def spiders(self): @@ -178,6 +179,7 @@ class CrawlerRunner(object): def _done(result): self.crawlers.discard(crawler) self._active.discard(d) + self.bootstrap_failed |= not getattr(crawler, 'spider', None) return result return d.addBoth(_done) diff --git a/tests/test_commands.py b/tests/test_commands.py index cb1301c95..7d9071b64 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import inspect import os import sys import subprocess @@ -17,6 +18,7 @@ from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class ProjectTest(unittest.TestCase): @@ -220,6 +222,16 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_run_fail_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + ret = proc.returncode + self.assertNotEqual(ret, 0) + + def test_run_good_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + ret = proc.returncode + self.assertEqual(ret, 0) + def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=('-s', 'LOG_LEVEL=INFO')) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ba0d709ff..d3b80f460 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,9 @@ import tempfile import warnings import unittest +from twisted.internet import defer +import twisted.trial.unittest + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings @@ -11,9 +14,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle - class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -181,3 +184,62 @@ class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_None(self): runner = CrawlerProcess() self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + + +class ExceptionSpider(scrapy.Spider): + name = 'exception' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + raise ValueError('Exception in from_crawler method') + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful_for_several(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + self.assertEqual(runner.bootstrap_failed, True) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed_for_several(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + yield runner.crawl(NoRequestsSpider) + + self.assertEqual(runner.bootstrap_failed, True) From 7a601d76de7adc37571815a7d08f84e1e26f7507 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 19 Jun 2018 10:51:55 +0200 Subject: [PATCH 156/179] fix typo extractred --> extracted --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..1b8d121a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -156,7 +156,7 @@ attributes with ``FormRequest``. **Please also note that link extractors do not canonicalize URLs by default anymore.** This was puzzling users every now and then, and it's not what -browsers do in fact, so we removed that extra transformation on extractred +browsers do in fact, so we removed that extra transformation on extracted links. For those of you wanting more control on the ``Referer:`` header that Scrapy From 88bd067912ee94e2a6d2e1ba5b0d5db1241b7621 Mon Sep 17 00:00:00 2001 From: Grammy Jiang Date: Wed, 20 Jun 2018 16:56:46 +0800 Subject: [PATCH 157/179] fix the test case name of HttpProxyMiddleware --- tests/test_downloadermiddleware_httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 17be875c1..537126613 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,7 @@ from scrapy.settings import Settings spider = Spider('foo') -class TestDefaultHeadersMiddleware(TestCase): +class TestHttpProxyMiddleware(TestCase): failureException = AssertionError From 9ad3af9d88bcefa18394c1cb2c833902ac20c533 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Sat, 23 Jun 2018 17:31:54 +0800 Subject: [PATCH 158/179] Update requirements.txt make the version of ipython less than 6.0 in python 2.7 --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index c1576a2e7..790f29d34 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -10,4 +10,4 @@ brotlipy testfixtures # optional for shell wrapper tests bpython -ipython +ipython<6.0 From fac1b2f3516f3db6ce669664943336c08d95d0aa Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 03:23:47 +0500 Subject: [PATCH 159/179] TST remove workaround for old Pillow versions which don't support BytesIO --- tests/test_pipeline_images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 03c6d8059..a7c652959 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,8 +1,8 @@ -import os +import io import hashlib import random import warnings -from tempfile import mkdtemp, TemporaryFile +from tempfile import mkdtemp from shutil import rmtree from twisted.trial import unittest @@ -401,8 +401,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + def _create_image(format, *a, **kw): - buf = TemporaryFile() + buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf) From 45f67eb64d54f2ac9fcd69233d7bddbdcec88a37 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jun 2018 14:51:01 +0500 Subject: [PATCH 160/179] TST exclude lxml==4.2.2 from tests, as it doesn't play well with Pillow --- tests/constraints.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/constraints.txt b/tests/constraints.txt index 3bc30de15..e59e68b3f 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,2 @@ Twisted!=18.4.0 +lxml!=4.2.2 \ No newline at end of file diff --git a/tox.ini b/tox.ini index c2fa9af28..82348eb24 100644 --- a/tox.ini +++ b/tox.ini @@ -67,6 +67,7 @@ commands = [testenv:py34] basepython = python3.4 deps = + -ctests/constraints.txt -rrequirements-py3.txt # Extras Pillow From 8782901fc865ca6f505bcd1a7d17314076507028 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Jun 2018 01:11:15 +0500 Subject: [PATCH 161/179] [MRG+1] TST test agains latest pypy (#3309) pypy3 is not upgraded, as tests segfault with pypy3 6.0 for some reason --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6635f5d3b..065f23805 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: install: - | if [ "$TOXENV" = "pypy" ]; then - export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable" wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" From f11d65f7d66cf2d8560c707c4bb2b76079d45e5f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Jun 2018 18:34:11 +0500 Subject: [PATCH 162/179] TST make it clear which requirements are Python 2-only * rename requirements.txt to requirements-py2.txt, to make it clear they are Python 2-only * make requirements-py3.txt consistent with requirements-py2.txt --- requirements.txt => requirements-py2.txt | 4 ++-- requirements-py3.txt | 3 +++ tests/{requirements.txt => requirements-py2.txt} | 0 tox.ini | 8 ++++---- 4 files changed, 9 insertions(+), 6 deletions(-) rename requirements.txt => requirements-py2.txt (100%) rename tests/{requirements.txt => requirements-py2.txt} (100%) diff --git a/requirements.txt b/requirements-py2.txt similarity index 100% rename from requirements.txt rename to requirements-py2.txt index 2a94d742d..03b33d02d 100644 --- a/requirements.txt +++ b/requirements-py2.txt @@ -2,9 +2,9 @@ Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.17.0 queuelib +w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -service_identity parsel>=1.4 +service_identity diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..d76d9412f 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -4,4 +4,7 @@ pyOpenSSL>=0.13.1 cssselect>=0.9 queuelib>=1.1.1 w3lib>=1.17.0 +six>=1.5.2 +PyDispatcher>=2.0.5 +parsel>=1.4 service_identity diff --git a/tests/requirements.txt b/tests/requirements-py2.txt similarity index 100% rename from tests/requirements.txt rename to tests/requirements-py2.txt diff --git a/tox.ini b/tox.ini index 82348eb24..ee40983de 100644 --- a/tox.ini +++ b/tox.ini @@ -9,13 +9,13 @@ envlist = py27 [testenv] deps = -ctests/constraints.txt - -rrequirements.txt + -rrequirements-py2.txt # Extras botocore google-cloud-storage Pillow != 3.0.0 leveldb - -rtests/requirements.txt + -rtests/requirements-py2.txt passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -35,7 +35,7 @@ deps = Pillow==2.3.0 cssselect==0.9.1 zope.interface==4.0.5 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:jessie] # https://packages.debian.org/en/jessie/python/ @@ -50,7 +50,7 @@ deps = Pillow==2.6.1 cssselect==0.9.1 zope.interface==4.1.1 - -rtests/requirements.txt + -rtests/requirements-py2.txt [testenv:trunk] basepython = python2.7 From d05c8677c5079b88db4405fe2bed83dc437c9204 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Wed, 4 Jul 2018 02:58:43 +0800 Subject: [PATCH 163/179] [MRG+1] change the bad smell code (#3304) Change the bad smell code --- scrapy/downloadermiddlewares/httpproxy.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 0d5320bf8..1dd47359f 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,14 +1,13 @@ import base64 +from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -from six.moves.urllib.parse import unquote try: from urllib2 import _parse_proxy except ImportError: from urllib.request import _parse_proxy -from six.moves.urllib.parse import urlunparse -from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -17,8 +16,8 @@ class HttpProxyMiddleware(object): def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding self.proxies = {} - for type, url in getproxies().items(): - self.proxies[type] = self._get_proxy(url, type) + for type_, url in getproxies().items(): + self.proxies[type_] = self._get_proxy(url, type_) @classmethod def from_crawler(cls, crawler): From 74ce1561542dac9be5d1363a4a3e623653855e05 Mon Sep 17 00:00:00 2001 From: chainly <1258626769@qq.com> Date: Wed, 4 Jul 2018 03:00:59 +0800 Subject: [PATCH 164/179] add item_error to be catchable (#3256) --- docs/topics/signals.rst | 23 +++++++++++++++++++++++ scrapy/core/scraper.py | 3 +++ scrapy/signals.py | 1 + tests/pipelines.py | 6 ++++++ tests/test_engine.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index cf1588df8..d40c0e1df 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -135,6 +135,29 @@ item_dropped to be dropped :type exception: :exc:`~scrapy.exceptions.DropItem` exception +item_error +------------ + +.. signal:: item_error +.. function:: item_error(item, response, spider, failure) + + Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises + an exception), except :exc:`~scrapy.exceptions.DropItem` exception. + + This signal supports returning deferreds from their handlers. + + :param item: the item dropped from the :ref:`topics-item-pipeline` + :type item: dict or :class:`~scrapy.item.Item` object + + :param response: the response being processed when the exception was raised + :type response: :class:`~scrapy.http.Response` object + + :param spider: the spider which raised the exception + :type spider: :class:`~scrapy.spiders.Spider` object + + :param failure: the exception raised as a Twisted `Failure`_ object + :type failure: `Failure`_ object + spider_closed ------------- diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c08e37367..ee1e95a0c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -232,6 +232,9 @@ class Scraper(object): logger.error('Error processing %(item)s', {'item': item}, exc_info=failure_to_exc_info(output), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) else: logkws = self.logformatter.scraped(output, response, spider) logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) diff --git a/scrapy/signals.py b/scrapy/signals.py index de0886fb6..e36c27203 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ response_received = object() response_downloaded = object() item_scraped = object() item_dropped = object() +item_error = object() # for backwards compatibility stats_spider_opened = spider_opened diff --git a/tests/pipelines.py b/tests/pipelines.py index ddfbc7a99..7e2895a5c 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -9,3 +9,9 @@ class ZeroDivisionErrorPipeline(object): def process_item(self, item, spider): return item + + +class ProcessWithZeroDivisionErrorPipiline(object): + + def process_item(self, item, spider): + 1/0 diff --git a/tests/test_engine.py b/tests/test_engine.py index 04113ddcf..719c0c60c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -74,6 +74,14 @@ class DictItemsSpider(TestSpider): item_cls = dict +class ItemZeroDivisionErrorSpider(TestSpider): + custom_settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300, + } + } + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -95,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.itemerror = [] self.itemresp = [] self.signals_catched = {} self.spider_class = spider_class @@ -112,6 +121,7 @@ class CrawlerRun(object): self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) @@ -136,6 +146,9 @@ class CrawlerRun(object): u = urlparse(url) return u.path + def item_error(self, item, response, spider, failure): + self.itemerror.append((item, response, spider, failure)) + def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) @@ -175,6 +188,10 @@ class EngineTest(unittest.TestCase): self._assert_scheduled_requests(urls_to_visit=7) self._assert_dropped_requests() + self.run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield self.run.run() + self._assert_items_error() + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] @@ -209,6 +226,20 @@ class EngineTest(unittest.TestCase): if self.run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) + def _assert_items_error(self): + self.assertEqual(2, len(self.run.itemerror)) + for item, response, spider, failure in self.run.itemerror: + self.assertEqual(failure.value.__class__, ZeroDivisionError) + self.assertEqual(spider, self.run.spider) + + self.assertEqual(item['url'], response.url) + if 'item1.html' in item['url']: + self.assertEqual('Item 1 name', item['name']) + self.assertEqual('100', item['price']) + if 'item2.html' in item['url']: + self.assertEqual('Item 2 name', item['name']) + self.assertEqual('200', item['price']) + def _assert_scraped_items(self): self.assertEqual(2, len(self.run.itemresp)) for item, response in self.run.itemresp: From 6f5c39d65f3e5d74c292f2143b4f84c7f43b155a Mon Sep 17 00:00:00 2001 From: Oz T Date: Wed, 4 Jul 2018 00:22:24 +0300 Subject: [PATCH 165/179] Fix for CSV export unnecessary blank lines problem on Windows (#3039) --- scrapy/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 07f43b494..695c74fec 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -214,7 +214,8 @@ class CsvItemExporter(BaseItemExporter): file, line_buffering=False, write_through=True, - encoding=self.encoding + encoding=self.encoding, + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) if six.PY3 else file self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True From d4c7cc848b83c2ec38ea90b76369885daba7375c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:19:43 +0500 Subject: [PATCH 166/179] remove backwards compatibility shims for relocated modules --- .coveragerc | 9 --------- conftest.py | 9 +-------- scrapy/command.py | 7 ------- scrapy/contrib/__init__.py | 0 scrapy/contrib/closespider.py | 7 ------- scrapy/contrib/corestats.py | 7 ------- scrapy/contrib/debug.py | 7 ------- .../contrib/downloadermiddleware/__init__.py | 0 .../contrib/downloadermiddleware/ajaxcrawl.py | 7 ------- scrapy/contrib/downloadermiddleware/chunked.py | 7 ------- scrapy/contrib/downloadermiddleware/cookies.py | 7 ------- .../downloadermiddleware/decompression.py | 7 ------- .../downloadermiddleware/defaultheaders.py | 7 ------- .../downloadermiddleware/downloadtimeout.py | 7 ------- .../contrib/downloadermiddleware/httpauth.py | 7 ------- .../contrib/downloadermiddleware/httpcache.py | 7 ------- .../downloadermiddleware/httpcompression.py | 7 ------- .../contrib/downloadermiddleware/httpproxy.py | 7 ------- .../contrib/downloadermiddleware/redirect.py | 7 ------- scrapy/contrib/downloadermiddleware/retry.py | 7 ------- .../contrib/downloadermiddleware/robotstxt.py | 7 ------- scrapy/contrib/downloadermiddleware/stats.py | 7 ------- .../contrib/downloadermiddleware/useragent.py | 7 ------- scrapy/contrib/exporter/__init__.py | 8 -------- scrapy/contrib/feedexport.py | 7 ------- scrapy/contrib/httpcache.py | 7 ------- scrapy/contrib/linkextractors/__init__.py | 7 ------- scrapy/contrib/linkextractors/htmlparser.py | 7 ------- scrapy/contrib/linkextractors/lxmlhtml.py | 7 ------- scrapy/contrib/linkextractors/regex.py | 7 ------- scrapy/contrib/linkextractors/sgml.py | 7 ------- scrapy/contrib/loader/__init__.py | 7 ------- scrapy/contrib/loader/common.py | 7 ------- scrapy/contrib/loader/processor.py | 7 ------- scrapy/contrib/logstats.py | 7 ------- scrapy/contrib/memdebug.py | 7 ------- scrapy/contrib/memusage.py | 7 ------- scrapy/contrib/pipeline/__init__.py | 7 ------- scrapy/contrib/pipeline/files.py | 7 ------- scrapy/contrib/pipeline/images.py | 7 ------- scrapy/contrib/pipeline/media.py | 7 ------- scrapy/contrib/spidermiddleware/__init__.py | 0 scrapy/contrib/spidermiddleware/depth.py | 7 ------- scrapy/contrib/spidermiddleware/httperror.py | 7 ------- scrapy/contrib/spidermiddleware/offsite.py | 7 ------- scrapy/contrib/spidermiddleware/referer.py | 7 ------- scrapy/contrib/spidermiddleware/urllength.py | 7 ------- scrapy/contrib/spiders/__init__.py | 7 ------- scrapy/contrib/spiders/crawl.py | 7 ------- scrapy/contrib/spiders/feed.py | 7 ------- scrapy/contrib/spiders/init.py | 7 ------- scrapy/contrib/spiders/sitemap.py | 7 ------- scrapy/contrib/spiderstate.py | 7 ------- scrapy/contrib/statsmailer.py | 7 ------- scrapy/contrib/throttle.py | 7 ------- scrapy/contrib_exp/__init__.py | 0 .../downloadermiddleware/__init__.py | 0 .../downloadermiddleware/decompression.py | 7 ------- scrapy/contrib_exp/iterators.py | 6 ------ scrapy/dupefilter.py | 7 ------- scrapy/linkextractor.py | 7 ------- scrapy/spider.py | 7 ------- scrapy/squeue.py | 7 ------- scrapy/statscol.py | 7 ------- scrapy/utils/decorator.py | 7 ------- scrapy/utils/deprecate.py | 18 ------------------ 66 files changed, 1 insertion(+), 441 deletions(-) delete mode 100644 scrapy/command.py delete mode 100644 scrapy/contrib/__init__.py delete mode 100644 scrapy/contrib/closespider.py delete mode 100644 scrapy/contrib/corestats.py delete mode 100644 scrapy/contrib/debug.py delete mode 100644 scrapy/contrib/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py delete mode 100644 scrapy/contrib/downloadermiddleware/chunked.py delete mode 100644 scrapy/contrib/downloadermiddleware/cookies.py delete mode 100644 scrapy/contrib/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py delete mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py delete mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py delete mode 100644 scrapy/contrib/downloadermiddleware/redirect.py delete mode 100644 scrapy/contrib/downloadermiddleware/retry.py delete mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py delete mode 100644 scrapy/contrib/downloadermiddleware/stats.py delete mode 100644 scrapy/contrib/downloadermiddleware/useragent.py delete mode 100644 scrapy/contrib/exporter/__init__.py delete mode 100644 scrapy/contrib/feedexport.py delete mode 100644 scrapy/contrib/httpcache.py delete mode 100644 scrapy/contrib/linkextractors/__init__.py delete mode 100644 scrapy/contrib/linkextractors/htmlparser.py delete mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py delete mode 100644 scrapy/contrib/linkextractors/regex.py delete mode 100644 scrapy/contrib/linkextractors/sgml.py delete mode 100644 scrapy/contrib/loader/__init__.py delete mode 100644 scrapy/contrib/loader/common.py delete mode 100644 scrapy/contrib/loader/processor.py delete mode 100644 scrapy/contrib/logstats.py delete mode 100644 scrapy/contrib/memdebug.py delete mode 100644 scrapy/contrib/memusage.py delete mode 100644 scrapy/contrib/pipeline/__init__.py delete mode 100644 scrapy/contrib/pipeline/files.py delete mode 100644 scrapy/contrib/pipeline/images.py delete mode 100644 scrapy/contrib/pipeline/media.py delete mode 100644 scrapy/contrib/spidermiddleware/__init__.py delete mode 100644 scrapy/contrib/spidermiddleware/depth.py delete mode 100644 scrapy/contrib/spidermiddleware/httperror.py delete mode 100644 scrapy/contrib/spidermiddleware/offsite.py delete mode 100644 scrapy/contrib/spidermiddleware/referer.py delete mode 100644 scrapy/contrib/spidermiddleware/urllength.py delete mode 100644 scrapy/contrib/spiders/__init__.py delete mode 100644 scrapy/contrib/spiders/crawl.py delete mode 100644 scrapy/contrib/spiders/feed.py delete mode 100644 scrapy/contrib/spiders/init.py delete mode 100644 scrapy/contrib/spiders/sitemap.py delete mode 100644 scrapy/contrib/spiderstate.py delete mode 100644 scrapy/contrib/statsmailer.py delete mode 100644 scrapy/contrib/throttle.py delete mode 100644 scrapy/contrib_exp/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/__init__.py delete mode 100644 scrapy/contrib_exp/downloadermiddleware/decompression.py delete mode 100644 scrapy/contrib_exp/iterators.py delete mode 100644 scrapy/dupefilter.py delete mode 100644 scrapy/linkextractor.py delete mode 100644 scrapy/spider.py delete mode 100644 scrapy/squeue.py delete mode 100644 scrapy/statscol.py delete mode 100644 scrapy/utils/decorator.py diff --git a/.coveragerc b/.coveragerc index 3105409ba..aeadccb25 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,13 +7,4 @@ omit = scrapy/conf.py scrapy/stats.py scrapy/project.py - scrapy/utils/decorator.py - scrapy/statscol.py - scrapy/squeue.py scrapy/log.py - scrapy/dupefilter.py - scrapy/command.py - scrapy/linkextractor.py - scrapy/spider.py - scrapy/contrib/* - scrapy/contrib_exp/* diff --git a/conftest.py b/conftest.py index 8b4faf8fc..c733db646 100644 --- a/conftest.py +++ b/conftest.py @@ -13,19 +13,12 @@ collect_ignore = [ "scrapy/conf.py", "scrapy/stats.py", "scrapy/project.py", - "scrapy/utils/decorator.py", - "scrapy/statscol.py", - "scrapy/squeue.py", "scrapy/log.py", - "scrapy/dupefilter.py", - "scrapy/command.py", - "scrapy/linkextractor.py", - "scrapy/spider.py", # not a test, but looks like a test "scrapy/utils/testsite.py", -] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp") +] if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0): collect_ignore += _py_files("scrapy/xlib/tx") diff --git a/scrapy/command.py b/scrapy/command.py deleted file mode 100644 index 3e1219bbc..000000000 --- a/scrapy/command.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.command` is deprecated, " - "use `scrapy.commands` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.commands import * diff --git a/scrapy/contrib/__init__.py b/scrapy/contrib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py deleted file mode 100644 index 9c52c418f..000000000 --- a/scrapy/contrib/closespider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.closespider` is deprecated, " - "use `scrapy.extensions.closespider` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.closespider import * diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py deleted file mode 100644 index 2f5354239..000000000 --- a/scrapy/contrib/corestats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.corestats` is deprecated, " - "use `scrapy.extensions.corestats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.corestats import * diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py deleted file mode 100644 index a38f059ce..000000000 --- a/scrapy/contrib/debug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.debug` is deprecated, " - "use `scrapy.extensions.debug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.debug import * diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py deleted file mode 100644 index 90ebc46b6..000000000 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, " - "use `scrapy.downloadermiddlewares.ajaxcrawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.ajaxcrawl import * diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py deleted file mode 100644 index 1322c9083..000000000 --- a/scrapy/contrib/downloadermiddleware/chunked.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, " - "use `scrapy.downloadermiddlewares.chunked` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.chunked import * diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py deleted file mode 100644 index bad970690..000000000 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, " - "use `scrapy.downloadermiddlewares.cookies` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.cookies import * diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py deleted file mode 100644 index a541aa61e..000000000 --- a/scrapy/contrib/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import * diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py deleted file mode 100644 index cf023dc8f..000000000 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, " - "use `scrapy.downloadermiddlewares.defaultheaders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.defaultheaders import * diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py deleted file mode 100644 index 84bd06acf..000000000 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, " - "use `scrapy.downloadermiddlewares.downloadtimeout` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.downloadtimeout import * diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py deleted file mode 100644 index a37ffa0dc..000000000 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.httpauth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py deleted file mode 100644 index f5f068204..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcache import * diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py deleted file mode 100644 index 8a52ec50b..000000000 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, " - "use `scrapy.downloadermiddlewares.httpcompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpcompression import * diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py deleted file mode 100644 index d94d85076..000000000 --- a/scrapy/contrib/downloadermiddleware/httpproxy.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, " - "use `scrapy.downloadermiddlewares.httpproxy` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.httpproxy import * diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py deleted file mode 100644 index 824eee8ae..000000000 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, " - "use `scrapy.downloadermiddlewares.redirect` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.redirect import * diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py deleted file mode 100644 index aafe0f508..000000000 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, " - "use `scrapy.downloadermiddlewares.retry` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.retry import * diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py deleted file mode 100644 index 408f760a0..000000000 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, " - "use `scrapy.downloadermiddlewares.robotstxt` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.robotstxt import * diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py deleted file mode 100644 index fa84a8206..000000000 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, " - "use `scrapy.downloadermiddlewares.stats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.stats import * diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py deleted file mode 100644 index 893d5241c..000000000 --- a/scrapy/contrib/downloadermiddleware/useragent.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, " - "use `scrapy.downloadermiddlewares.useragent` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.useragent import * diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py deleted file mode 100644 index 12adaaddd..000000000 --- a/scrapy/contrib/exporter/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.exporter` is deprecated, " - "use `scrapy.exporters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.exporters import * -from scrapy.exporters import PythonItemExporter diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py deleted file mode 100644 index 19651998a..000000000 --- a/scrapy/contrib/feedexport.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, " - "use `scrapy.extensions.feedexport` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.feedexport import * diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py deleted file mode 100644 index 196372fcb..000000000 --- a/scrapy/contrib/httpcache.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, " - "use `scrapy.extensions.httpcache` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.httpcache import * diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py deleted file mode 100644 index 976658df3..000000000 --- a/scrapy/contrib/linkextractors/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py deleted file mode 100644 index ff03da98f..000000000 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, " - "use `scrapy.linkextractors.htmlparser` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.htmlparser import * diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py deleted file mode 100644 index fc2b7de3c..000000000 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, " - "use `scrapy.linkextractors.lxmlhtml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.lxmlhtml import * diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py deleted file mode 100644 index 97bda29c1..000000000 --- a/scrapy/contrib/linkextractors/regex.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, " - "use `scrapy.linkextractors.regex` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.regex import * diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py deleted file mode 100644 index a5a598208..000000000 --- a/scrapy/contrib/linkextractors/sgml.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, " - "use `scrapy.linkextractors.sgml` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors.sgml import * diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py deleted file mode 100644 index 2b9453e18..000000000 --- a/scrapy/contrib/loader/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader` is deprecated, " - "use `scrapy.loader` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader import * diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py deleted file mode 100644 index a59b2b7b1..000000000 --- a/scrapy/contrib/loader/common.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, " - "use `scrapy.loader.common` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.common import * diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py deleted file mode 100644 index da7e484a5..000000000 --- a/scrapy/contrib/loader/processor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, " - "use `scrapy.loader.processors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.loader.processors import * diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py deleted file mode 100644 index 62bc9b860..000000000 --- a/scrapy/contrib/logstats.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.logstats` is deprecated, " - "use `scrapy.extensions.logstats` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.logstats import * diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py deleted file mode 100644 index 4f6e4760e..000000000 --- a/scrapy/contrib/memdebug.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, " - "use `scrapy.extensions.memdebug` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memdebug import * diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py deleted file mode 100644 index e13bd78f3..000000000 --- a/scrapy/contrib/memusage.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.memusage` is deprecated, " - "use `scrapy.extensions.memusage` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.memusage import * diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py deleted file mode 100644 index aedf34a3f..000000000 --- a/scrapy/contrib/pipeline/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, " - "use `scrapy.pipelines` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines import * diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py deleted file mode 100644 index cd1238b5d..000000000 --- a/scrapy/contrib/pipeline/files.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, " - "use `scrapy.pipelines.files` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.files import * diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py deleted file mode 100644 index 4f5ce4c40..000000000 --- a/scrapy/contrib/pipeline/images.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, " - "use `scrapy.pipelines.images` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.images import * diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py deleted file mode 100644 index 4b4fea560..000000000 --- a/scrapy/contrib/pipeline/media.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, " - "use `scrapy.pipelines.media` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.pipelines.media import * diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py deleted file mode 100644 index 718803148..000000000 --- a/scrapy/contrib/spidermiddleware/depth.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, " - "use `scrapy.spidermiddlewares.depth` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.depth import * diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py deleted file mode 100644 index e39fb3f56..000000000 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, " - "use `scrapy.spidermiddlewares.httperror` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.httperror import * diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py deleted file mode 100644 index a5ed9ea7e..000000000 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, " - "use `scrapy.spidermiddlewares.offsite` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.offsite import * diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py deleted file mode 100644 index fdf8d6659..000000000 --- a/scrapy/contrib/spidermiddleware/referer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, " - "use `scrapy.spidermiddlewares.referer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.referer import * diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py deleted file mode 100644 index 5e51add59..000000000 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, " - "use `scrapy.spidermiddlewares.urllength` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spidermiddlewares.urllength import * diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py deleted file mode 100644 index 56780533b..000000000 --- a/scrapy/contrib/spiders/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py deleted file mode 100644 index d20a8bb16..000000000 --- a/scrapy/contrib/spiders/crawl.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, " - "use `scrapy.spiders.crawl` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.crawl import * diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py deleted file mode 100644 index 5eea9a062..000000000 --- a/scrapy/contrib/spiders/feed.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, " - "use `scrapy.spiders.feed` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.feed import * diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py deleted file mode 100644 index 6d1ec0aa9..000000000 --- a/scrapy/contrib/spiders/init.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, " - "use `scrapy.spiders.init` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.init import * diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py deleted file mode 100644 index 2ad231fd8..000000000 --- a/scrapy/contrib/spiders/sitemap.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, " - "use `scrapy.spiders.sitemap` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders.sitemap import * diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py deleted file mode 100644 index 06afc8bfc..000000000 --- a/scrapy/contrib/spiderstate.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, " - "use `scrapy.extensions.spiderstate` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.spiderstate import * diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py deleted file mode 100644 index f9c9a37f5..000000000 --- a/scrapy/contrib/statsmailer.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, " - "use `scrapy.extensions.statsmailer` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.statsmailer import * diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py deleted file mode 100644 index d5c234871..000000000 --- a/scrapy/contrib/throttle.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib.throttle` is deprecated, " - "use `scrapy.extensions.throttle` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.throttle import * diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py deleted file mode 100644 index 1f8490587..000000000 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.downloadermiddlewares.decompression` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py deleted file mode 100644 index c59f47bcc..000000000 --- a/scrapy/contrib_exp/iterators.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.iterators import xmliter_lxml diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py deleted file mode 100644 index 232d96288..000000000 --- a/scrapy/dupefilter.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.dupefilter` is deprecated, " - "use `scrapy.dupefilters` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.dupefilters import * diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py deleted file mode 100644 index b744aff8e..000000000 --- a/scrapy/linkextractor.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.linkextractor` is deprecated, " - "use `scrapy.linkextractors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.linkextractors import * diff --git a/scrapy/spider.py b/scrapy/spider.py deleted file mode 100644 index 56a5a0a0b..000000000 --- a/scrapy/spider.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.spider` is deprecated, " - "use `scrapy.spiders` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.spiders import * diff --git a/scrapy/squeue.py b/scrapy/squeue.py deleted file mode 100644 index a4a3f4238..000000000 --- a/scrapy/squeue.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.squeue` is deprecated, " - "use `scrapy.squeues` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.squeues import * diff --git a/scrapy/statscol.py b/scrapy/statscol.py deleted file mode 100644 index b4ddcce28..000000000 --- a/scrapy/statscol.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.statscol` is deprecated, " - "use `scrapy.statscollectors` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.statscollectors import * diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py deleted file mode 100644 index e8c8eae39..000000000 --- a/scrapy/utils/decorator.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.utils.decorator` is deprecated, " - "use `scrapy.utils.decorators` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.utils.decorators import * diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f76161a68..8c72cc556 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -124,25 +124,7 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ - ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'), - ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'), - ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'), - ('scrapy.contrib.exporter.', 'scrapy.exporters.'), - ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'), - ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'), - ('scrapy.contrib.loader.', 'scrapy.loader.'), - ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'), - ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'), - ('scrapy.contrib.spiders.', 'scrapy.spiders.'), - ('scrapy.contrib.', 'scrapy.extensions.'), - ('scrapy.command.', 'scrapy.commands.'), - ('scrapy.dupefilter.', 'scrapy.dupefilters.'), - ('scrapy.linkextractor.', 'scrapy.linkextractors.'), ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spider.', 'scrapy.spiders.'), - ('scrapy.squeue.', 'scrapy.squeues.'), - ('scrapy.statscol.', 'scrapy.statscollectors.'), - ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'), ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 36453348fad9babc96558ab10af9b2942eb5431e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:23:37 +0500 Subject: [PATCH 167/179] remove ancient modules kept only for error messages --- .coveragerc | 2 -- conftest.py | 2 -- scrapy/project.py | 17 ----------------- scrapy/stats.py | 8 -------- 4 files changed, 29 deletions(-) delete mode 100644 scrapy/project.py delete mode 100644 scrapy/stats.py diff --git a/.coveragerc b/.coveragerc index aeadccb25..1fde07e7e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,4 @@ omit = tests/* scrapy/xlib/* scrapy/conf.py - scrapy/stats.py - scrapy/project.py scrapy/log.py diff --git a/conftest.py b/conftest.py index c733db646..2d015f5e9 100644 --- a/conftest.py +++ b/conftest.py @@ -11,8 +11,6 @@ def _py_files(folder): collect_ignore = [ # deprecated or moved modules "scrapy/conf.py", - "scrapy/stats.py", - "scrapy/project.py", "scrapy/log.py", # not a test, but looks like a test diff --git a/scrapy/project.py b/scrapy/project.py deleted file mode 100644 index d8973a6c7..000000000 --- a/scrapy/project.py +++ /dev/null @@ -1,17 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("""scrapy.project usage has become obsolete. - -If you want to get the Scrapy crawler from your extension, middleware or -pipeline implement the `from_crawler` class method (or look up for extending -components that have already done it, such as spiders). - -For example: - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler)""") diff --git a/scrapy/stats.py b/scrapy/stats.py deleted file mode 100644 index 710601430..000000000 --- a/scrapy/stats.py +++ /dev/null @@ -1,8 +0,0 @@ - -""" -Obsolete module, kept for giving a meaningful error message when trying to -import. -""" - -raise ImportError("scrapy.stats usage has become obsolete, use " - "`crawler.stats` attribute instead") From f531b66822491140740a6d86af2f3f11f0443d38 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 6 Jul 2018 03:28:01 +0500 Subject: [PATCH 168/179] SpiderManager shim is removed --- scrapy/interfaces.py | 4 ---- scrapy/spidermanager.py | 7 ------- scrapy/utils/deprecate.py | 1 - 3 files changed, 12 deletions(-) delete mode 100644 scrapy/spidermanager.py diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..89ad2b14f 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -16,7 +16,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py deleted file mode 100644 index 220257bb1..000000000 --- a/scrapy/spidermanager.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Backwards compatibility shim. Use scrapy.spiderloader instead. -""" -from scrapy.spiderloader import SpiderLoader -from scrapy.utils.deprecate import create_deprecated_class - -SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 8c72cc556..2d3db431d 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -125,7 +125,6 @@ def _clspath(cls, forced=None): DEPRECATION_RULES = [ ('scrapy.telnet.', 'scrapy.extensions.telnet.'), - ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'), ] From 722e1afcdb337bf11652167f02435c81fc68ecfb Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:21:19 +0300 Subject: [PATCH 169/179] Update ancient pytest on python 3 2.9 gives collection errors on python 3.7 due to PEP 479. --- tests/requirements-py3.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 51a25f5e5..8d9ce5231 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,6 @@ -pytest==2.9.2 +pytest==3.6.3 pytest-twisted -pytest-cov==2.2.1 +pytest-cov==2.5.1 testfixtures jmespath leveldb From 17e9914b8a12e5a96cc40016b74f301fb9835cbf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:09 +0300 Subject: [PATCH 170/179] Catch SyntaxError as well when importing manhole Also give a more detailed reason why telnet is not enabled (for the future). --- requirements-py3.txt | 2 +- scrapy/extensions/telnet.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index b941fd867..b38c4cc09 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -git+https://github.com/lopuhin/twisted.git@9384-remove-async-param +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index e78afa1fc..7cc8f823a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -12,7 +12,7 @@ try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True -except ImportError: +except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,7 +40,8 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured + raise NotConfigured('TelnetConsole not enabled: failed to import ' + 'required twisted modules.') self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From cf9399acc149cf5eafb2d00d310416ab2ba185e5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:26:56 +0300 Subject: [PATCH 171/179] Use python 3.7 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29f9f0065..f6ea670ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=docs - - python: 3.7-dev + - python: 3.7 env: TOXENV=py37 install: - | From 2773fe09e4b4fac51dbc06725f1fafb2e5d9a271 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:36:58 +0300 Subject: [PATCH 172/179] Make "docs" the last build, even though it still uses python3.6 for now --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6ea670ae..88c72b08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,10 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 3.6 - env: TOXENV=docs - python: 3.7 env: TOXENV=py37 + - python: 3.6 + env: TOXENV=docs install: - | if [ "$TOXENV" = "pypy" ]; then From f4f39057cbbfa4daf66f82061e57101b88d88d05 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:46:45 +0300 Subject: [PATCH 173/179] Make csviter work on python 3.7 PEP 479 does not allow for StopIteration in generators. Instead, handle it explicitly, also use a for loop which looks simpler. --- scrapy/utils/iterators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 73857b410..a12e14005 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): """ encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' - def _getrow(csv_r): - return [to_unicode(field, encoding) for field in next(csv_r)] + + def row_to_unicode(row_): + return [to_unicode(field, encoding) for field in row_] # Python 3 csv reader input object needs to return strings if six.PY3: @@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): csv_r = csv.reader(lines, **kwargs) if not headers: - headers = _getrow(csv_r) + try: + row = next(csv_r) + except StopIteration: + return + headers = row_to_unicode(row) - while True: - row = _getrow(csv_r) + for row in csv_r: + row = row_to_unicode(row) if len(row) != len(headers): logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " "should be: %(csvheader)d)", From b3cd12dc48592fb8b1d4c6883315fe2b341ca5c2 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 12:53:40 +0300 Subject: [PATCH 174/179] Try to get python3.7 by using xenial base and sudo See https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-401756442 and https://github.com/travis-ci/travis-ci/issues/9815#issuecomment-402045581 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 88c72b08e..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + dist: xenial + sudo: true - python: 3.6 env: TOXENV=docs install: From 92b504eae5379dadade2d78efba4e54b201cbd93 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 13:43:36 +0300 Subject: [PATCH 175/179] Fix telnet warnings in tests Disable telnet console if it's not available, else we'll get an extra warning about failure to enable it, and tests will fail. --- tests/test_crawler.py | 6 ++++-- tests/test_utils_log.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d3b80f460..6a8e11363 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import tempfile import warnings import unittest @@ -14,8 +13,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle +from scrapy.extensions import telnet + class BaseCrawlerTest(unittest.TestCase): @@ -100,6 +100,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file.name, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 45527b03b..742e04803 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter, LogCounterHandler, StreamLogger) from scrapy.utils.test import get_crawler +from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): @@ -65,10 +66,14 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): + settings = {'LOG_LEVEL': 'WARNING'} + if not telnet.TWISTED_CONCH_AVAILABLE: + # disable it to avoid the extra warning + settings['TELNETCONSOLE_ENABLED'] = False self.logger = logging.getLogger('test') self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(settings_dict={'LOG_LEVEL': 'WARNING'}) + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) From 4f6778aa7332aecd15f7672a0852d1f49596969b Mon Sep 17 00:00:00 2001 From: nyov Date: Mon, 9 Jul 2018 17:16:31 +0000 Subject: [PATCH 176/179] Remove deprecated CrawlerSettings class and Settings attributes --- docs/news.rst | 9 +++++ scrapy/settings/__init__.py | 48 -------------------------- tests/test_settings/__init__.py | 60 +-------------------------------- 3 files changed, 10 insertions(+), 107 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1b8d121a1..633e5c72f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,15 @@ Release notes ============= +Scrapy 1.6.0 (unreleased) +------------------------- + +Cleanups +~~~~~~~~ + +* Remove deprecated ``CrawlerSettings`` class. +* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + Scrapy 1.5.0 (2017-12-29) ------------------------- diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 28446a372..7d6d20164 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -6,7 +6,6 @@ from collections import MutableMapping from importlib import import_module from pprint import pformat -from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning from . import default_settings @@ -405,30 +404,6 @@ class BaseSettings(MutableMapping): else: p.text(pformat(self.copy_to_dict())) - @property - def overrides(self): - warnings.warn("`Settings.overrides` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='cmdline')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._overrides - except AttributeError: - self._overrides = o = _DictProxy(self, 'cmdline') - return o - - @property - def defaults(self): - warnings.warn("`Settings.defaults` attribute is deprecated and won't " - "be supported in Scrapy 0.26, use " - "`Settings.set(name, value, priority='default')` instead", - category=ScrapyDeprecationWarning, stacklevel=2) - try: - o = self._defaults - except AttributeError: - self._defaults = o = _DictProxy(self, 'default') - return o - class _DictProxy(MutableMapping): @@ -479,29 +454,6 @@ class Settings(BaseSettings): self.update(values, priority) -class CrawlerSettings(Settings): - - def __init__(self, settings_module=None, **kw): - self.settings_module = settings_module - Settings.__init__(self, **kw) - - def __getitem__(self, opt_name): - if opt_name in self.overrides: - return self.overrides[opt_name] - if self.settings_module and hasattr(self.settings_module, opt_name): - return getattr(self.settings_module, opt_name) - if opt_name in self.defaults: - return self.defaults[opt_name] - return Settings.__getitem__(self, opt_name) - - def __str__(self): - return "" % self.settings_module - -CrawlerSettings = create_deprecated_class( - 'CrawlerSettings', CrawlerSettings, - new_class_path='scrapy.settings.Settings') - - def iter_default_settings(): """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 863684075..1dbacbea3 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -3,8 +3,7 @@ import unittest import warnings from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, - CrawlerSettings, SETTINGS_PRIORITIES, - get_settings_priority) + SETTINGS_PRIORITIES, get_settings_priority) from tests import mock from . import default_settings @@ -341,35 +340,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertTrue(frozencopy.frozen) self.assertIsNot(frozencopy, self.settings) - def test_deprecated_attribute_overrides(self): - self.settings.set('BAR', 'fuz', priority='cmdline') - with warnings.catch_warnings(record=True) as w: - self.settings.overrides['BAR'] = 'foo' - self.assertIn("Settings.overrides", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.overrides.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.overrides) - - self.settings.overrides.update(BAR='bus') - self.assertEqual(self.settings.get('BAR'), 'bus') - self.assertEqual(self.settings.overrides.get('BAR'), 'bus') - - self.settings.overrides.setdefault('BAR', 'fez') - self.assertEqual(self.settings.get('BAR'), 'bus') - - self.settings.overrides.setdefault('FOO', 'fez') - self.assertEqual(self.settings.get('FOO'), 'fez') - self.assertEqual(self.settings.overrides.get('FOO'), 'fez') - - def test_deprecated_attribute_defaults(self): - self.settings.set('BAR', 'fuz', priority='default') - with warnings.catch_warnings(record=True) as w: - self.settings.defaults['BAR'] = 'foo' - self.assertIn("Settings.defaults", str(w[0].message)) - self.assertEqual(self.settings.get('BAR'), 'foo') - self.assertEqual(self.settings.defaults.get('BAR'), 'foo') - self.assertIn('BAR', self.settings.defaults) - class SettingsTest(unittest.TestCase): @@ -422,33 +392,5 @@ class SettingsTest(unittest.TestCase): self.assertEqual(mydict['key'], 'val') -class CrawlerSettingsTest(unittest.TestCase): - - def test_deprecated_crawlersettings(self): - def _get_settings(settings_dict=None): - settings_module = type('SettingsModuleMock', (object,), settings_dict or {}) - return CrawlerSettings(settings_module) - - with warnings.catch_warnings(record=True) as w: - settings = _get_settings() - self.assertIn("CrawlerSettings is deprecated", str(w[0].message)) - - # test_global_defaults - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180) - - # test_defaults - settings.defaults['DOWNLOAD_TIMEOUT'] = '99' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99) - - # test_settings_module - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3) - - # test_overrides - settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'}) - settings.overrides['DOWNLOAD_TIMEOUT'] = '15' - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15) - - if __name__ == "__main__": unittest.main() From 9428a4a3aa4c679c15eb2c606de7b49ad832ee6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 9 Jul 2018 21:03:26 +0300 Subject: [PATCH 177/179] More visible telnet conch message Capture traceback when trying to import required twisted modules, print it in case telnet is enabled, and mention settings variable that can be used to supress the message. Thanks @kmike! --- scrapy/extensions/telnet.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 7cc8f823a..3024ddfaa 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -6,6 +6,7 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging +import traceback from twisted.internet import protocol try: @@ -13,6 +14,7 @@ try: from twisted.conch.insults import insults TWISTED_CONCH_AVAILABLE = True except (ImportError, SyntaxError): + _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False from scrapy.exceptions import NotConfigured @@ -40,8 +42,9 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured('TelnetConsole not enabled: failed to import ' - 'required twisted modules.') + raise NotConfigured( + 'TELNETCONSOLE_ENABLED setting is True but required twisted ' + 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK) self.crawler = crawler self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] From c86213317daf25aec04f3afc69c327a43033987f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 12 Jul 2018 02:10:24 +0500 Subject: [PATCH 178/179] 1.5.1 release notes --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 633e5c72f..01016e2e6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,6 +12,22 @@ Cleanups * Remove deprecated ``CrawlerSettings`` class. * Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes. + +Scrapy 1.5.1 (2018-07-12) +------------------------- + +This is a maintenance release with important bug fixes, but no new features: + +* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy + is fixed (:issue:`3281`); +* skipping of TLS validation errors is improved (:issue:`3166`); +* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`); +* testing fixes (:issue:`3092`, :issue:`3263`); +* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`, + :issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`, + :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`, + :issue:`3294`). + Scrapy 1.5.0 (2017-12-29) ------------------------- From c61e8a617f1291bfcbe56d54a2f80f8fb79b7ddb Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Fri, 13 Jul 2018 11:55:16 -0700 Subject: [PATCH 179/179] [doc] update default RETRY_HTTP_CODES --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfe4c13b4..8dbe249fa 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -882,7 +882,7 @@ precedence over the :setting:`RETRY_TIMES` setting. RETRY_HTTP_CODES ^^^^^^^^^^^^^^^^ -Default: ``[500, 502, 503, 504, 408]`` +Default: ``[500, 502, 503, 504, 522, 524, 408]`` Which HTTP response codes to retry. Other errors (DNS lookup issues, connections lost, etc) are always retried.