diff --git a/.travis.yml b/.travis.yml index dafa980bb..cf5b450db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,20 +15,26 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 - - python: 3.5 + - env: TOXENV=py + python: 3.5 - env: TOXENV=pinned python: 3.5 - env: TOXENV=asyncio python: 3.5.2 - - python: 3.6 + - env: TOXENV=py + python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - - python: 3.7 - - env: PYPI_RELEASE_JOB=true + - env: TOXENV=py + python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 + dist: bionic - env: TOXENV=extra-deps python: 3.8 + dist: bionic - env: TOXENV=asyncio python: 3.8 + dist: bionic install: - | if [ "$TOXENV" = "pypy3" ]; then diff --git a/pytest.ini b/pytest.ini index 4f3494e0e..053793853 100644 --- a/pytest.ini +++ b/pytest.ini @@ -35,19 +35,19 @@ flake8-ignore = scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E501 E128 + scrapy/commands/fetch.py E501 E128 scrapy/commands/genspider.py E128 E501 scrapy/commands/parse.py E128 E501 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 - scrapy/commands/startproject.py E127 E501 E128 + scrapy/commands/startproject.py E501 E128 scrapy/commands/version.py E501 E128 # scrapy/contracts scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 E127 + scrapy/core/engine.py E501 E128 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E128 scrapy/core/spidermw.py E501 E126 @@ -57,7 +57,7 @@ flake8-ignore = scrapy/core/downloader/tls.py E501 scrapy/core/downloader/webclient.py E501 E128 E126 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 E127 + scrapy/core/downloader/handlers/ftp.py E501 E128 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 scrapy/core/downloader/handlers/s3.py E501 E128 E126 @@ -73,7 +73,7 @@ flake8-ignore = scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 E123 + scrapy/extensions/closespider.py E501 E128 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 scrapy/extensions/httpcache.py E128 E501 @@ -85,10 +85,10 @@ flake8-ignore = scrapy/http/common.py E501 scrapy/http/cookies.py E501 scrapy/http/request/__init__.py E501 - scrapy/http/request/form.py E501 E123 + scrapy/http/request/form.py E501 scrapy/http/request/json_request.py E501 scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 E124 + scrapy/http/response/text.py E501 E128 # scrapy/linkextractors scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 @@ -99,7 +99,7 @@ flake8-ignore = scrapy/pipelines/__init__.py E501 scrapy/pipelines/files.py E116 E501 scrapy/pipelines/images.py E501 - scrapy/pipelines/media.py E125 E501 + scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 scrapy/selector/unified.py E501 E111 @@ -110,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 + scrapy/spidermiddlewares/referer.py E501 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -124,7 +124,7 @@ flake8-ignore = scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 - scrapy/utils/deprecate.py E128 E501 E127 + scrapy/utils/deprecate.py E501 scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 @@ -137,7 +137,7 @@ flake8-ignore = scrapy/utils/python.py E501 scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 - scrapy/utils/request.py E127 E501 + scrapy/utils/request.py E501 scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 @@ -169,10 +169,10 @@ flake8-ignore = scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 - tests/mockserver.py E401 E501 E126 E123 + tests/mockserver.py E501 E126 tests/pipelines.py F841 - tests/spiders.py E501 E127 - tests/test_closespider.py E501 E127 + tests/spiders.py E501 + tests/test_closespider.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 tests/test_command_shell.py E501 E128 @@ -181,34 +181,34 @@ flake8-ignore = tests/test_crawl.py E501 E741 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123 + tests/test_downloader_handlers.py E128 E501 E126 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126 - tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 + tests/test_downloadermiddleware_httpcompression.py E501 E126 + tests/test_downloadermiddleware_decompression.py E501 tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 E127 + tests/test_downloadermiddleware_redirect.py E501 E128 tests/test_downloadermiddleware_retry.py E501 E128 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 E124 - tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E128 E124 + tests/test_dupefilters.py E501 E741 E128 + tests/test_engine.py E501 E128 + tests/test_exporters.py E501 E128 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 + tests/test_http_request.py E402 E501 E128 E128 E126 tests/test_http_response.py E501 E128 tests/test_item.py E128 F841 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 E124 + tests/test_linkextractors.py E501 E128 tests/test_loader.py E501 E741 E128 E117 - tests/test_logformatter.py E128 E501 E122 + tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E501 E128 E126 @@ -219,14 +219,14 @@ flake8-ignore = tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py E501 E127 + tests/test_scheduler.py E501 E126 + tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 + tests/test_spidermiddleware_httperror.py E128 E501 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121 + tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 @@ -235,16 +235,16 @@ flake8-ignore = tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 E128 - tests/test_utils_iterators.py E501 E128 E129 + tests/test_utils_iterators.py E501 E128 tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 - tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E125 E501 E126 E123 - tests/test_webclient.py E501 E128 E122 E402 E123 E126 + tests/test_utils_sitemap.py E128 E501 + tests/test_utils_url.py E501 E501 E126 + tests/test_webclient.py E501 E128 E402 E126 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 432cb1831..94b55c347 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -94,11 +94,12 @@ class FTPDownloadHandler: def gotClient(self, client, request, filepath): self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol)\ - .addCallbacks(callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,)) + return client.retrieveFile(filepath, protocol).addCallbacks( + callback=self._build_response, + callbackArgs=(request, protocol), + errback=self._failed, + errbackArgs=(request,), + ) def _build_response(self, result, request, protocol): self.result = result diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 77d71846e..324d21716 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -230,8 +230,7 @@ class ExecutionEngine: def _downloaded(self, response, slot, request, spider): slot.remove_request(request) - return self.download(response, spider) \ - if isinstance(response, Request) else response + return self.download(response, spider) if isinstance(response, Request) else response def _download(self, request, spider): slot = self.slot diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index e3f212bef..812844c0a 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -20,7 +20,7 @@ class CloseSpider: 'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'), 'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'), 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), - } + } if not any(self.close_on.values()): raise NotConfigured diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af02c8484..cd4e3373f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -178,12 +178,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [ - el for el in form.xpath( - 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', - namespaces={"re": "http://exslt.org/regular-expressions"}) - ] + clickables = list(form.xpath( + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', + namespaces={"re": "http://exslt.org/regular-expressions"} + )) if not clickables: return diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ae365db5b..a9066986b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -230,6 +230,20 @@ class GCSFilesStore: bucket, prefix = uri[5:].split('/', 1) self.bucket = client.bucket(bucket) self.prefix = prefix + permissions = self.bucket.test_iam_permissions( + ['storage.objects.get', 'storage.objects.create'] + ) + if 'storage.objects.get' not in permissions: + logger.warning( + "No 'storage.objects.get' permission for GSC bucket %(bucket)s. " + "Checking if files are up to date will be impossible. Files will be downloaded every time.", + {'bucket': bucket} + ) + if 'storage.objects.create' not in permissions: + logger.error( + "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!", + {'bucket': bucket} + ) def stat_file(self, path, info): def _onsuccess(blob): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 8a0636264..aa65f4f0e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -43,8 +43,7 @@ class MediaPipeline: if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, - settings=None): + def _key_for_pipe(self, key, base_class_name=None, settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -55,8 +54,11 @@ class MediaPipeline: """ class_name = self.__class__.__name__ formatted_key = "{}_{}".format(class_name.upper(), key) - if class_name == base_class_name or not base_class_name \ - or (settings and not settings.get(formatted_key)): + if ( + not base_class_name + or class_name == base_class_name + or settings and not settings.get(formatted_key) + ): return key return formatted_key diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 36001d982..3dbea5fee 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'): stacklevel=3) -def create_deprecated_class(name, new_class, clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from " - "deprecated class {old}, please inherit " - "from {new}.", - instance_warn_message="{cls} is deprecated, " - "instantiate {new} instead."): +def create_deprecated_class( + name, + new_class, + clsdict=None, + warn_category=ScrapyDeprecationWarning, + warn_once=True, + old_class_path=None, + new_class_path=None, + subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message="{cls} is deprecated, instantiate {new} instead." +): """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b8c140a7e..12c03d78e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): """ if include_headers: - include_headers = tuple(to_bytes(h.lower()) - for h in sorted(include_headers)) + include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) cache_key = (include_headers, keep_fragments) if cache_key not in cache: diff --git a/tests/spiders.py b/tests/spiders.py index 284c77829..33d5d02e1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, \ - 'All start requests consumed before any download happened' + assert self.seedsseen, 'All start requests consumed before any download happened' def parse(self, response): self.seedsseen.append(response.meta.get('seed')) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a56425b7..5ec5e2989 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,8 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'\ - .format(name=crawler.spider.exception_cls.__name__) + key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b4144ea1d..9151278a5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -311,14 +311,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 29d06bab4..f93bce8ef 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -822,11 +822,15 @@ class S3TestCase(unittest.TestCase): def test_request_signing2(self): # puts an object into the johnsmith bucket. date = 'Tue, 27 Mar 2007 21:15:45 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ - 'Content-Type': 'image/jpeg', - 'Date': date, - 'Content-Length': '94328', - }) + req = Request( + 's3://johnsmith/photos/puppy.jpg', + method='PUT', + headers={ + 'Content-Type': 'image/jpeg', + 'Date': date, + 'Content-Length': '94328', + }, + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -906,11 +910,10 @@ class S3TestCase(unittest.TestCase): # ensure that spaces are quoted properly before signing date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request( - ("s3://johnsmith/photos/my puppy.jpg" - "?response-content-disposition=my puppy.jpg"), + "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg", method='GET', headers={'Date': date}, - ) + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual( @@ -1090,8 +1093,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): self.assertEqual(response.text, 'A brief note') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "US-ASCII") request = Request("data:,A%20brief%20note") @@ -1100,8 +1102,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:;charset=iso-8859-7,%be%d3%be") @@ -1119,8 +1120,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 77b35a8c3..dbae4d3ae 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - assert isinstance(new, XmlResponse), \ - 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) def test_plain_response(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 106ca3360..e86568bfb 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding SAMPLEDIR = join(tests_datadir, 'compressed') FORMAT = { - 'gzip': ('html-gzip.bin', 'gzip'), - 'x-gzip': ('html-gzip.bin', 'gzip'), - 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), - 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), - 'br': ('html-br.bin', 'br') - } + 'gzip': ('html-gzip.bin', 'gzip'), + 'x-gzip': ('html-gzip.bin', 'gzip'), + 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), + 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + 'br': ('html-br.bin', 'br'), +} class HttpCompressionTest(TestCase): @@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase): body = sample.read() headers = { - 'Server': 'Yaws/1.49 Yet Another Web Server', - 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', - 'Content-Length': len(body), - 'Content-Type': 'text/html', - 'Content-Encoding': contentencoding, - } + 'Server': 'Yaws/1.49 Yet Another Web Server', + 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', + 'Content-Length': len(body), + 'Content-Type': 'text/html', + 'Content-Encoding': contentencoding, + } response = Response('http://scrapytest.org/', body=body, headers=headers) response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 053e26fc3..551e124ab 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase): rsp = Response(url, headers={'Location': url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) self.assertIs(r, rsp) - _test_passthrough(Request(url, meta={'handle_httpstatus_list': - [404, 301, 302]})) + _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]})) _test_passthrough(Request(url, meta={'handle_httpstatus_all': True})) def test_latin1_location(self): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index ea0e664be..7426107c1 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -197,8 +197,7 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html', - headers={'Referer': 'http://scrapytest.org/INDEX.html'} - ) + headers={'Referer': 'http://scrapytest.org/INDEX.html'}) dupefilter.log(r1, spider) dupefilter.log(r2, spider) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 160912847..0f9dafcaa 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -342,20 +342,22 @@ class XmlItemExporterTest(BaseItemExporterTest): i2 = dict(name=u'bar', age=i1) i3 = TestItem(name=u'buz', age=i2) - self.assertExportResult(i3, - b'\n' - b'' - b'' - b'' - b'' - b'22' - b'foo\xc2\xa3hoo' - b'' - b'bar' - b'' - b'buz' - b'' - b'' + self.assertExportResult( + i3, + b"""\n + + + + + 22 + foo\xc2\xa3hoo + + bar + + buz + + + """ ) def test_nested_list_item(self): @@ -363,31 +365,35 @@ class XmlItemExporterTest(BaseItemExporterTest): i2 = dict(name=u'bar', v2={"egg": ["spam"]}) i3 = TestItem(name=u'buz', age=[i1, i2]) - self.assertExportResult(i3, - b'\n' - b'' - b'' - b'' - b'foo' - b'barspam' - b'' - b'buz' - b'' - b'' + self.assertExportResult( + i3, + b"""\n + + + + foo + barspam + + buz + + + """ ) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() - self.assertExportResult(item, - b'\n' - b'' - b'' - b'3.14' - b'False' - b'22' - b'' - b'' - b'' + self.assertExportResult( + item, + b"""\n + + + 3.14 + False + 22 + + + + """ ) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index cc2cddda4..3b6d119a9 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -399,8 +399,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_bytes(self): data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} - r2 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'latin1') self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') @@ -408,8 +407,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_textual_data(self): data = {'price': u'£ 100'} - r3 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') @@ -469,7 +467,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html", encoding='latin1', - ) + ) req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 53968e60e..68e8514ba 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -413,24 +413,30 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True)] - ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False), + Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + ] + ) def test_link_wrong_href(self): html = b""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 00568aee9..930a5dd99 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -46,13 +46,13 @@ class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', - SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' - ) + SCHEDULER_DEBUG=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', + ) super(MockCrawler, self).__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -305,10 +305,12 @@ class StartUrlsSpider(Spider): class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( - StartUrlsSpider, - {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', - 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'} - ) + spidercls=StartUrlsSpider, + settings_dict={ + 'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + }, + ) @defer.inlineCallbacks def tearDown(self): @@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase): def _incompatible(self): settings = dict( - SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', - CONCURRENT_REQUESTS_PER_IP=1 - ) + SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', + CONCURRENT_REQUESTS_PER_IP=1, + ) crawler = Crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name='spider') diff --git a/tests/test_selector.py b/tests/test_selector.py index 09c2546fb..65b0f5860 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), - [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index dacd0147f..6b61df56f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() @@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 742adc64f..41589177a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -478,32 +478,32 @@ class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) def test_valid_name_casevariants(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a85087619..46aaaecbc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -93,8 +93,8 @@ class XmliterTestCase(unittest.TestCase): # with bytes XmlResponse(url="http://example.com", body=body.encode('utf-8')), # Unicode body needs encoding information - XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - + XmlResponse(url="http://example.com", body=body, encoding='utf-8'), + ): attrs = [] for x in self.xmliter(r, u'þingflokkur'): attrs.append((x.attrib['id'], diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index db323ab31..08b215434 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -58,10 +58,13 @@ class SitemapTest(unittest.TestCase): """) - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_wrong_ns(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -80,10 +83,13 @@ class SitemapTest(unittest.TestCase): """) - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_wrong_ns2(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -103,10 +109,13 @@ class SitemapTest(unittest.TestCase): """) assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'loc': 'http://www.example.com/2', 'lastmod': ''}, + ] + ) def test_sitemap_urls_from_robots(self): robots = """User-agent: * @@ -195,11 +204,19 @@ Disallow: /forum/active/ """) - self.assertEqual(list(s), [ - {'loc': 'http://www.example.com/english/', - 'alternate': ['http://www.example.com/deutsch/', 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/english/'] - } - ]) + self.assertEqual( + list(s), + [ + { + 'loc': 'http://www.example.com/english/', + 'alternate': [ + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/english/', + ], + } + ] + ) def test_xml_entity_expansion(self): s = Sitemap(b""" diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 72a16e9b1..1f8388957 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase): class AddHttpIfNoScheme(unittest.TestCase): def test_add_scheme(self): - self.assertEqual(add_http_if_no_scheme('www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('example.com'), - 'http://example.com') + self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') def test_path(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_port(self): - self.assertEqual(add_http_if_no_scheme('www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('www.example.com:80'), + 'http://www.example.com:80') def test_fragment(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_query(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_username_password(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com'), + 'http://username:password@www.example.com') def test_complete_url(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_http(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') def test_preserve_http_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('http://example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('http://example.com'), + 'http://example.com') def test_preserve_http_path(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_preserve_http_port(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com:80'), + 'http://www.example.com:80') def test_preserve_http_fragment(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_preserve_http_query(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_preserve_http_username_password(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com'), + 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_protocol_relative(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com'), - 'http://www.example.com') + self.assertEqual( + add_http_if_no_scheme('//www.example.com'), 'http://www.example.com') def test_protocol_relative_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('//example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('//example.com'), 'http://example.com') def test_protocol_relative_path(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_protocol_relative_port(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('//www.example.com:80'), + 'http://www.example.com:80') def test_protocol_relative_fragment(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_protocol_relative_query(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_protocol_relative_username_password(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com'), + 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_https(self): - self.assertEqual(add_http_if_no_scheme('https://www.example.com'), - 'https://www.example.com') + self.assertEqual( + add_http_if_no_scheme('https://www.example.com'), + 'https://www.example.com') def test_preserve_ftp(self): - self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), - 'ftp://www.example.com') + self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com') class GuessSchemeTest(unittest.TestCase): @@ -202,41 +218,49 @@ def create_skipped_scheme_t(args): return do_expected -for k, args in enumerate([ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), +for k, args in enumerate( + [ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - ], start=1): + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + ], + start=1, +): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate([ - (r'C:\absolute\path\to\a\file.html', 'file://', - 'Windows filepath are not supported for scrapy shell'), - ], start=1): +for k, args in enumerate( + [ + ( + r'C:\absolute\path\to\a\file.html', + 'file://', + 'Windows filepath are not supported for scrapy shell', + ), + ], + start=1, +): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) @@ -272,7 +296,7 @@ class StripUrl(unittest.TestCase): ('http://www.example.com', True, 'http://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) def test_credentials(self): @@ -285,7 +309,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_credentials_encoded_delims(self): @@ -304,7 +328,7 @@ class StripUrl(unittest.TestCase): # password: "user@domain.com" ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_default_ports_creds_off(self): @@ -332,7 +356,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i), o) def test_default_ports(self): @@ -360,7 +384,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): @@ -388,7 +412,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): @@ -404,7 +428,7 @@ class StripUrl(unittest.TestCase): ('https://username:password@www.example.com:443/index.html', 'https://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(i, origin_only=True), o) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4abebbfb..b657c7ab6 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -53,29 +53,29 @@ class ParseUrlTestCase(unittest.TestCase): def testParse(self): lip = '127.0.0.1' tests = ( - ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), + ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), - ("http://127.0.0.1", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), - ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), - ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), - ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), + ("http://127.0.0.1", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), + ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), + ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), + ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), - ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), - ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), + ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), + ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), + ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), - ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), - ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), - ) + ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), + ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), + ) for url, test in tests: test = tuple( @@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers={ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - })) + }, + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" @@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers=Headers({ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - }))) + }), + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" diff --git a/tox.ini b/tox.ini index 697328ebd..2102fc602 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py3 +envlist = security,flake8,py minversion = 1.7.0 [testenv]