diff --git a/.travis.yml b/.travis.yml index 6bde973f4..cb54c8008 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,19 +15,25 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=pypy3 - - python: 3.5 + - env: TOXENV=py + python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=asyncio python: 3.5.2 - - python: 3.6 - - python: 3.7 - - env: PYPI_RELEASE_JOB=true + - env: TOXENV=py + python: 3.6 + - env: TOXENV=py + python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 + dist: bionic - env: TOXENV=extra-deps python: 3.8 + dist: bionic - env: TOXENV=asyncio python: 3.8 + dist: bionic install: - | if [ "$TOXENV" = "pypy3" ]; then diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 18f81838f..f06d9db3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``). .. note:: HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, - unless you use Twisted<11.1, or if you really want to use HTTP/1.0 - and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme - accordingly, i.e. to - ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. + unless you really want to use HTTP/1.0 and override + :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, + i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY @@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things). Scrapy also has another context factory class that you can set, ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, which uses the platform's certificates to validate remote endpoints. - **This is only available if you use Twisted>=14.0.** If you do use a custom ContextFactory, make sure its ``__init__`` method accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping @@ -494,10 +492,6 @@ This setting must be one of these string values: - ``'TLSv1.2'``: forces TLS version 1.2 - ``'SSLv3'``: forces SSL version 3 (**not recommended**) -.. note:: - - We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 - or above (Twisted>=14.0 if you can). .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -660,8 +654,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_maxsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_WARNSIZE DOWNLOAD_WARNSIZE @@ -679,8 +671,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_warnsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_FAIL_ON_DATALOSS DOWNLOAD_FAIL_ON_DATALOSS diff --git a/pytest.ini b/pytest.ini index 292dbce41..3d2608b03 100644 --- a/pytest.ini +++ b/pytest.ini @@ -35,62 +35,62 @@ flake8-ignore = scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E501 E128 + scrapy/commands/fetch.py E501 E128 scrapy/commands/genspider.py E128 E501 scrapy/commands/parse.py E128 E501 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 - scrapy/commands/startproject.py E127 E501 E128 + scrapy/commands/startproject.py E501 E128 scrapy/commands/version.py E501 E128 # scrapy/contracts - scrapy/contracts/__init__.py E501 W504 + scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 E127 + scrapy/core/engine.py E501 E128 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 W504 - scrapy/core/spidermw.py E501 E126 + scrapy/core/scraper.py E501 E128 + scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 E126 + scrapy/core/downloader/contextfactory.py E501 E128 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 E126 + scrapy/core/downloader/webclient.py E501 E128 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 E127 + scrapy/core/downloader/handlers/ftp.py E501 E128 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E128 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 - scrapy/downloadermiddlewares/httpcache.py E501 E126 + scrapy/downloadermiddlewares/httpcache.py E501 scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 - scrapy/downloadermiddlewares/redirect.py E501 W504 - scrapy/downloadermiddlewares/retry.py E501 E126 + scrapy/downloadermiddlewares/redirect.py E501 + scrapy/downloadermiddlewares/retry.py E501 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 E123 + scrapy/extensions/closespider.py E501 E128 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 scrapy/extensions/httpcache.py E128 E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 - scrapy/extensions/telnet.py E501 W504 + scrapy/extensions/telnet.py E501 scrapy/extensions/throttle.py E501 # scrapy/http scrapy/http/common.py E501 scrapy/http/cookies.py E501 scrapy/http/request/__init__.py E501 - scrapy/http/request/form.py E501 E123 + scrapy/http/request/form.py E501 scrapy/http/request/json_request.py E501 scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 E124 + scrapy/http/response/text.py E501 E128 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E501 E402 W504 + scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader scrapy/loader/__init__.py E501 E128 @@ -99,7 +99,7 @@ flake8-ignore = scrapy/pipelines/__init__.py E501 scrapy/pipelines/files.py E116 E501 scrapy/pipelines/images.py E501 - scrapy/pipelines/media.py E125 E501 + scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 scrapy/selector/unified.py E501 E111 @@ -110,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W504 + scrapy/spidermiddlewares/referer.py E501 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -124,8 +124,8 @@ flake8-ignore = scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 - scrapy/utils/deprecate.py E128 E501 E127 - scrapy/utils/gz.py E501 W504 + scrapy/utils/deprecate.py E501 + scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 @@ -137,7 +137,7 @@ flake8-ignore = scrapy/utils/python.py E501 scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 - scrapy/utils/request.py E127 E501 + scrapy/utils/request.py E501 scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 @@ -164,15 +164,15 @@ flake8-ignore = scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py F841 E501 E126 + scrapy/spiderloader.py F841 E501 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 - tests/mockserver.py E401 E501 E126 E123 + tests/mockserver.py E501 tests/pipelines.py F841 - tests/spiders.py E501 E127 - tests/test_closespider.py E501 E127 + tests/spiders.py E501 + tests/test_closespider.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 tests/test_command_shell.py E501 E128 @@ -181,37 +181,37 @@ flake8-ignore = tests/test_crawl.py E501 E741 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123 + tests/test_downloader_handlers.py E128 E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126 - tests/test_downloadermiddleware_decompression.py E127 + tests/test_downloadermiddleware_cookies.py E741 E501 E128 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 + tests/test_downloadermiddleware_httpcompression.py E501 + tests/test_downloadermiddleware_decompression.py E501 tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 E127 - tests/test_downloadermiddleware_retry.py E501 E128 E126 + tests/test_downloadermiddleware_redirect.py E501 E128 + tests/test_downloadermiddleware_retry.py E501 E128 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 E124 - tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E128 E124 + tests/test_dupefilters.py E501 E741 E128 + tests/test_engine.py E501 E128 + tests/test_exporters.py E501 E128 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 + tests/test_http_request.py E402 E501 E128 E128 tests/test_http_response.py E501 E128 tests/test_item.py E128 F841 E501 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 E124 + tests/test_linkextractors.py E501 E128 tests/test_loader.py E501 E741 E128 E117 - tests/test_logformatter.py E128 E501 E122 + tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 E126 + tests/test_pipeline_crawl.py E501 E128 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 tests/test_pipeline_media.py E501 E741 E128 @@ -219,14 +219,14 @@ flake8-ignore = tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py E501 E127 + tests/test_scheduler.py E501 + tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 + tests/test_spidermiddleware_httperror.py E128 E501 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121 + tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 @@ -234,17 +234,17 @@ flake8-ignore = tests/test_utils_datatypes.py E402 E501 tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 W504 - tests/test_utils_iterators.py E501 E128 E129 + tests/test_utils_http.py E501 E128 + tests/test_utils_iterators.py E501 E128 tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 - tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E125 E501 E126 E123 - tests/test_webclient.py E501 E128 E122 E402 E123 E126 + tests/test_utils_sitemap.py E128 E501 + tests/test_utils_url.py E501 E501 + tests/test_webclient.py E501 E128 E402 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 41d4f25b2..5af3831a2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -17,10 +17,10 @@ class ContractsManager: self.contracts[contract.name] = contract def tested_methods_from_spidercls(self, spidercls): + is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): - if (callable(value) and value.__doc__ and - re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + if callable(value) and value.__doc__ and is_method(value.__doc__): methods.append(key) return methods diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 6e023ebcc..ab73e12c8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): # # This means that a website like https://www.cacert.org will be rejected # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={ - 'method': self._ssl_method, - }) + return optionsForClientTLS( + hostname=hostname.decode("ascii"), + trustRoot=platformTrust(), + extraCertificateOptions={'method': self._ssl_method}, + ) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 432cb1831..94b55c347 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -94,11 +94,12 @@ class FTPDownloadHandler: def gotClient(self, client, request, filepath): self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol)\ - .addCallbacks(callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,)) + return client.retrieveFile(filepath, protocol).addCallbacks( + callback=self._build_response, + callbackArgs=(request, protocol), + errback=self._failed, + errbackArgs=(request,), + ) def _build_response(self, result, request, protocol): self.result = result diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 40a1fa48e..8f63ad974 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -100,11 +100,12 @@ class S3DownloadHandler: url=url, headers=awsrequest.headers.items()) else: signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body) + method=request.method, + bucket=bucket, + key=unquote(p.path), + query_args=unquote(p.query), + headers=request.headers, + data=request.body, + ) request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a90a77b2b..355045d74 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." % - (self.factory.url, self.factory.timeout))) + defer.TimeoutError("Getting %s took longer than %s seconds." + % (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 77d71846e..324d21716 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -230,8 +230,7 @@ class ExecutionEngine: def _downloaded(self, response, slot, request, spider): slot.remove_request(request) - return self.download(response, spider) \ - if isinstance(response, Request) else response + return self.download(response, spider) if isinstance(response, Request) else response def _download(self, request, spider): slot = self.slot diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 09ee8377e..b32afb8e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + def process_response(self, request, response, spider): - if (request.meta.get('dont_redirect', False) or - response.status in getattr(spider, 'handle_httpstatus_list', []) or - response.status in request.meta.get('handle_httpstatus_list', []) or - request.meta.get('handle_httpstatus_all', False)): + if ( + request.meta.get('dont_redirect', False) + or response.status in getattr(spider, 'handle_httpstatus_list', []) + or response.status in request.meta.get('handle_httpstatus_list', []) + or request.meta.get('handle_httpstatus_all', False) + ): return response allowed_status = (301, 302, 303, 307, 308) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index bbf5fca05..6d11af5b2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages. import logging from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index e3f212bef..812844c0a 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -20,7 +20,7 @@ class CloseSpider: 'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'), 'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'), 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), - } + } if not any(self.close_on.values()): raise NotConfigured diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 04ffd7235..1663604e7 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username.encode('utf8') and - credentials.checkPassword(self.password.encode('utf8'))): + if not ( + credentials.username == self.username.encode('utf8') + and credentials.checkPassword(self.password.encode('utf8')) + ): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af02c8484..cd4e3373f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -178,12 +178,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [ - el for el in form.xpath( - 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', - namespaces={"re": "http://exslt.org/regular-expressions"}) - ] + clickables = list(form.xpath( + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', + namespaces={"re": "http://exslt.org/regular-expressions"} + )) if not clickables: return diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d0b5066b6..ae019c70f 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -61,8 +61,7 @@ class FilteringLinkExtractor: def __new__(cls, *args, **kwargs): from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if (issubclass(cls, FilteringLinkExtractor) and - not issubclass(cls, LxmlLinkExtractor)): + if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ae365db5b..a9066986b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -230,6 +230,20 @@ class GCSFilesStore: bucket, prefix = uri[5:].split('/', 1) self.bucket = client.bucket(bucket) self.prefix = prefix + permissions = self.bucket.test_iam_permissions( + ['storage.objects.get', 'storage.objects.create'] + ) + if 'storage.objects.get' not in permissions: + logger.warning( + "No 'storage.objects.get' permission for GSC bucket %(bucket)s. " + "Checking if files are up to date will be impossible. Files will be downloaded every time.", + {'bucket': bucket} + ) + if 'storage.objects.create' not in permissions: + logger.error( + "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!", + {'bucket': bucket} + ) def stat_file(self, path, info): def _onsuccess(blob): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 8a0636264..aa65f4f0e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -43,8 +43,7 @@ class MediaPipeline: if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, - settings=None): + def _key_for_pipe(self, key, base_class_name=None, settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -55,8 +54,11 @@ class MediaPipeline: """ class_name = self.__class__.__name__ formatted_key = "{}_{}".format(class_name.upper(), key) - if class_name == base_class_name or not base_class_name \ - or (settings and not settings.get(formatted_key)): + if ( + not base_class_name + or class_name == base_class_name + or settings and not settings.get(formatted_key) + ): return key return formatted_key diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 3be5aaec5..63da55718 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from collections import defaultdict import traceback import warnings +from collections import defaultdict from zope.interface import implementer @@ -16,6 +16,7 @@ class SpiderLoader: SpiderLoader is a class which locates and loads spiders in a Scrapy project. """ + def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') @@ -24,16 +25,21 @@ class SpiderLoader: self._load_all_spiders() def _check_name_duplicates(self): - dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( - module=mod, cls=cls, name=name) - for (mod, cls) in locations) - for name, locations in self._found.items() - if len(locations) > 1] + dupes = [] + for name, locations in self._found.items(): + dupes.extend([ + " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + for mod, cls in locations + if len(locations) > 1 + ]) + if dupes: - msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format( - "\n\n".join(dupes))) - warnings.warn(msg, UserWarning) + dupes_string = "\n\n".join(dupes) + warnings.warn( + "There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format(dupes_string), + category=UserWarning, + ) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -45,12 +51,15 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError as e: + except ImportError: if self.warn_only: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + warnings.warn( + "\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc() + ), + category=RuntimeWarning, + ) else: raise self._check_name_duplicates() @@ -73,8 +82,10 @@ class SpiderLoader: """ Return the list of spider names that can handle the given request. """ - return [name for name, cls in self._spiders.items() - if cls.handles_request(request)] + return [ + name for name, cls in self._spiders.items() + if cls.handles_request(request) + ] def list(self): """ diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 3784de885..434067b00 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): - if ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + if ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) @@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + elif ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 36001d982..3dbea5fee 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'): stacklevel=3) -def create_deprecated_class(name, new_class, clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from " - "deprecated class {old}, please inherit " - "from {new}.", - instance_warn_message="{cls} is deprecated, " - "instantiate {new} instead."): +def create_deprecated_class( + name, + new_class, + clsdict=None, + warn_category=ScrapyDeprecationWarning, + warn_once=True, + old_class_path=None, + new_class_path=None, + subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message="{cls} is deprecated, instantiate {new} instead." +): """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c291ae237..fbd7bd18f 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -52,8 +52,7 @@ def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return (_is_gzipped(ctype) or - (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') def gzip_magic_number(response): diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b8c140a7e..12c03d78e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): """ if include_headers: - include_headers = tuple(to_bytes(h.lower()) - for h in sorted(include_headers)) + include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) cache_key = (include_headers, keep_fragments) if cache_key not in cache: diff --git a/tests/spiders.py b/tests/spiders.py index 284c77829..33d5d02e1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, \ - 'All start requests consumed before any download happened' + assert self.seedsseen, 'All start requests consumed before any download happened' def parse(self, response): self.seedsseen.append(response.meta.get('seed')) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a56425b7..5ec5e2989 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,8 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'\ - .format(name=crawler.spider.exception_cls.__name__) + key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4144ea1d..9151278a5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 29d06bab4..f93bce8ef 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase): def test_request_signing2(self): # puts an object into the johnsmith bucket. date = 'Tue, 27 Mar 2007 21:15:45 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ - 'Content-Type': 'image/jpeg', - 'Date': date, - 'Content-Length': '94328', - }) + req = Request( + 's3://johnsmith/photos/puppy.jpg', + method='PUT', + headers={ + 'Content-Type': 'image/jpeg', + 'Date': date, + 'Content-Length': '94328', + }, + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase): # ensure that spaces are quoted properly before signing date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request( - ("s3://johnsmith/photos/my puppy.jpg" - "?response-content-disposition=my puppy.jpg"), + "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg", method='GET', headers={'Date': date}, - ) + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual( @@ -1090,8 +1093,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): self.assertEqual(response.text, 'A brief note') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "US-ASCII") request = Request("data:,A%20brief%20note") @@ -1100,8 +1102,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:;charset=iso-8859-7,%be%d3%be") @@ -1119,8 +1120,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index b686a14d6..f86c50f50 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase): def test_complex_cookies(self): # merge some cookies into jar - cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, - {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}] + cookies = [ + {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, + {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}, + ] req = Request('http://scrapytest.org/', cookies=cookies) self.mw.process_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 77b35a8c3..dbae4d3ae 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - assert isinstance(new, XmlResponse), \ - 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) def test_plain_response(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 106ca3360..e86568bfb 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding SAMPLEDIR = join(tests_datadir, 'compressed') FORMAT = { - 'gzip': ('html-gzip.bin', 'gzip'), - 'x-gzip': ('html-gzip.bin', 'gzip'), - 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), - 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), - 'br': ('html-br.bin', 'br') - } + 'gzip': ('html-gzip.bin', 'gzip'), + 'x-gzip': ('html-gzip.bin', 'gzip'), + 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), + 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + 'br': ('html-br.bin', 'br'), +} class HttpCompressionTest(TestCase): @@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase): body = sample.read() headers = { - 'Server': 'Yaws/1.49 Yet Another Web Server', - 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', - 'Content-Length': len(body), - 'Content-Type': 'text/html', - 'Content-Encoding': contentencoding, - } + 'Server': 'Yaws/1.49 Yet Another Web Server', + 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', + 'Content-Length': len(body), + 'Content-Type': 'text/html', + 'Content-Encoding': contentencoding, + } response = Response('http://scrapytest.org/', body=body, headers=headers) response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 053e26fc3..551e124ab 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase): rsp = Response(url, headers={'Location': url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) self.assertIs(r, rsp) - _test_passthrough(Request(url, meta={'handle_httpstatus_list': - [404, 301, 302]})) + _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]})) _test_passthrough(Request(url, meta={'handle_httpstatus_all': True})) def test_latin1_location(self): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 9c989977e..e118750e3 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,8 +1,14 @@ import unittest from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index ea0e664be..7426107c1 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html', - headers={'Referer': 'http://scrapytest.org/INDEX.html'} - ) + headers={'Referer': 'http://scrapytest.org/INDEX.html'}) dupefilter.log(r1, spider) dupefilter.log(r2, spider) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 160912847..0f9dafcaa 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest): i2 = dict(name=u'bar', age=i1) i3 = TestItem(name=u'buz', age=i2) - self.assertExportResult(i3, - b'\n' - b'' - b'' - b'' - b'' - b'22' - b'foo\xc2\xa3hoo' - b'' - b'bar' - b'' - b'buz' - b'' - b'' + self.assertExportResult( + i3, + b"""\n + + + + + 22 + foo\xc2\xa3hoo + + bar + + buz + + + """ ) def test_nested_list_item(self): @@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest): i2 = dict(name=u'bar', v2={"egg": ["spam"]}) i3 = TestItem(name=u'buz', age=[i1, i2]) - self.assertExportResult(i3, - b'\n' - b'' - b'' - b'' - b'foo' - b'barspam' - b'' - b'buz' - b'' - b'' + self.assertExportResult( + i3, + b"""\n + + + + foo + barspam + + buz + + + """ ) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() - self.assertExportResult(item, - b'\n' - b'' - b'' - b'3.14' - b'False' - b'22' - b'' - b'' - b'' + self.assertExportResult( + item, + b"""\n + + + 3.14 + False + 22 + + + + """ ) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index cc2cddda4..77da15ce6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -399,8 +399,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_bytes(self): data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} - r2 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'latin1') self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') @@ -408,8 +407,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_textual_data(self): data = {'price': u'£ 100'} - r3 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') @@ -469,7 +467,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html", encoding='latin1', - ) + ) req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) @@ -504,11 +502,13 @@ class FormRequestTest(RequestTest): def test_from_response_duplicate_form_key(self): response = _buildresponse( - '
', - url='http://www.example.com') - req = self.request_class.from_response(response, - method='GET', - formdata=(('foo', 'bar'), ('foo', 'baz'))) + '
', + url='http://www.example.com') + req = self.request_class.from_response( + response=response, + method='GET', + formdata=(('foo', 'bar'), ('foo', 'baz')), + ) self.assertEqual(urlparse(req.url).hostname, 'www.example.com') self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') @@ -532,9 +532,11 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}, - headers={"Accept-Encoding": "gzip,deflate"}) + req = self.request_class.from_response( + response=response, + formdata={'one': ['two', 'three'], 'six': 'seven'}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate') @@ -582,9 +584,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_method(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.method, 'GET') request = FormRequest.from_response(response, method='POST') @@ -592,9 +594,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_url(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.url, 'http://example.com/app') request = FormRequest.from_response(response, url='http://foo.bar/absolute') diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 53968e60e..68e8514ba 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -413,24 +413,30 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) def test_link_wrong_href(self): html = b""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 962c33144..24c516473 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): name = 'redirectedmedia' def _process_url(self, url): - return add_or_replace_parameter( - self.mockserver.url('/redirect-to'), - 'goto', url) + return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url) class FileDownloadCrawlTestCase(TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 00568aee9..930a5dd99 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -46,13 +46,13 @@ class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', - SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' - ) + SCHEDULER_DEBUG=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', + ) super(MockCrawler, self).__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -305,10 +305,12 @@ class StartUrlsSpider(Spider): class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( - StartUrlsSpider, - {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', - 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'} - ) + spidercls=StartUrlsSpider, + settings_dict={ + 'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + }, + ) @defer.inlineCallbacks def tearDown(self): @@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase): def _incompatible(self): settings = dict( - SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', - CONCURRENT_REQUESTS_PER_IP=1 - ) + SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', + CONCURRENT_REQUESTS_PER_IP=1, + ) crawler = Crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name='spider') diff --git a/tests/test_selector.py b/tests/test_selector.py index 09c2546fb..65b0f5860 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), - [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index dacd0147f..6b61df56f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() @@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 742adc64f..41589177a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) def test_valid_name_casevariants(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2fac3da1f..363b015a8 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual( + body, + "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" + ) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a85087619..46aaaecbc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase): # with bytes XmlResponse(url="http://example.com", body=body.encode('utf-8')), # Unicode body needs encoding information - XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - + XmlResponse(url="http://example.com", body=body, encoding='utf-8'), + ): attrs = [] for x in self.xmliter(r, u'þingflokkur'): attrs.append((x.attrib['id'], diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index db323ab31..08b215434 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase): """) - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_wrong_ns(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase): """) - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_wrong_ns2(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase): """) assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_urls_from_robots(self): robots = """User-agent: * @@ -195,11 +204,19 @@ Disallow: /forum/active/ """) - self.assertEqual(list(s), [ - {'loc': 'http://www.example.com/english/', - 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] - } - ]) + self.assertEqual( + list(s), + [ + { + 'loc': 'http://www.example.com/english/', + 'alternate': [ + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/english/', + ], + } + ] + ) def test_xml_entity_expansion(self): s = Sitemap(b""" diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 72a16e9b1..1f8388957 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase): class AddHttpIfNoScheme(unittest.TestCase): def test_add_scheme(self): - self.assertEqual(add_http_if_no_scheme('www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('example.com'), - 'http://example.com') + self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') def test_path(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_port(self): - self.assertEqual(add_http_if_no_scheme('www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('www.example.com:80'), + 'http://www.example.com:80') def test_fragment(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_query(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_username_password(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com'), + 'http://username:password@www.example.com') def test_complete_url(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_http(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') def test_preserve_http_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('http://example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('http://example.com'), + 'http://example.com') def test_preserve_http_path(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_preserve_http_port(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com:80'), + 'http://www.example.com:80') def test_preserve_http_fragment(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_preserve_http_query(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_preserve_http_username_password(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com'), + 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_protocol_relative(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com'), - 'http://www.example.com') + self.assertEqual( + add_http_if_no_scheme('//www.example.com'), 'http://www.example.com') def test_protocol_relative_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('//example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('//example.com'), 'http://example.com') def test_protocol_relative_path(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_protocol_relative_port(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('//www.example.com:80'), + 'http://www.example.com:80') def test_protocol_relative_fragment(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_protocol_relative_query(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_protocol_relative_username_password(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com'), + 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_https(self): - self.assertEqual(add_http_if_no_scheme('https://www.example.com'), - 'https://www.example.com') + self.assertEqual( + add_http_if_no_scheme('https://www.example.com'), + 'https://www.example.com') def test_preserve_ftp(self): - self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), - 'ftp://www.example.com') + self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com') class GuessSchemeTest(unittest.TestCase): @@ -202,41 +218,49 @@ def create_skipped_scheme_t(args): return do_expected -for k, args in enumerate([ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), +for k, args in enumerate( + [ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - ], start=1): + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + ], + start=1, +): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate([ - (r'C:\absolute\path\to\a\file.html', 'file://', - 'Windows filepath are not supported for scrapy shell'), - ], start=1): +for k, args in enumerate( + [ + ( + r'C:\absolute\path\to\a\file.html', + 'file://', + 'Windows filepath are not supported for scrapy shell', + ), + ], + start=1, +): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) @@ -272,7 +296,7 @@ class StripUrl(unittest.TestCase): ('http://www.example.com', True, 'http://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) def test_credentials(self): @@ -285,7 +309,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_credentials_encoded_delims(self): @@ -304,7 +328,7 @@ class StripUrl(unittest.TestCase): # password: "user@domain.com" ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_default_ports_creds_off(self): @@ -332,7 +356,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i), o) def test_default_ports(self): @@ -360,7 +384,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): @@ -388,7 +412,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): @@ -404,7 +428,7 @@ class StripUrl(unittest.TestCase): ('https://username:password@www.example.com:443/index.html', 'https://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(i, origin_only=True), o) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4abebbfb..307fadb5c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,6 +18,14 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks +from twisted.web.test.test_webclient import ( + ForeverTakingResource, + ErrorResource, + NoLengthResource, + HostHeaderResource, + PayloadResource, + BrokenDownloadResource, +) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -53,29 +61,29 @@ class ParseUrlTestCase(unittest.TestCase): def testParse(self): lip = '127.0.0.1' tests = ( - ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), + ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), - ("http://127.0.0.1", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), - ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), - ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), - ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), + ("http://127.0.0.1", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), + ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), + ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), + ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), - ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), - ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), + ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), + ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), + ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), - ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), - ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), - ) + ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), + ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), + ) for url, test in tests: test = tuple( @@ -149,7 +157,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers={ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - })) + }, + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" @@ -165,7 +174,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers=Headers({ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - }))) + }), + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" @@ -200,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): Headers({'Hello': ['World'], 'Foo': ['Bar']})) -from twisted.web.test.test_webclient import ForeverTakingResource, \ - ErrorResource, NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource - - class EncodingResource(resource.Resource): out_encoding = 'cp1251' diff --git a/tox.ini b/tox.ini index 697328ebd..2102fc602 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py3 +envlist = security,flake8,py minversion = 1.7.0 [testenv]