diff --git a/pytest.ini b/pytest.ini index b8a87b214..8177fedd3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -176,13 +176,13 @@ flake8-ignore = tests/test_command_shell.py E501 tests/test_commands.py E501 tests/test_contracts.py E501 - tests/test_crawl.py E501 E741 + tests/test_crawl.py E501 tests/test_crawler.py E501 tests/test_dependencies.py E501 tests/test_downloader_handlers.py E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 + tests/test_downloadermiddleware_cookies.py E501 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 @@ -193,7 +193,7 @@ flake8-ignore = tests/test_downloadermiddleware_retry.py E501 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 + tests/test_dupefilters.py E501 tests/test_engine.py E501 tests/test_exporters.py E501 tests/test_feedexport.py E501 @@ -201,6 +201,7 @@ flake8-ignore = tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 tests/test_http_response.py E501 + tests/test_item.py E501 tests/test_link.py E501 tests/test_linkextractors.py E501 tests/test_loader.py E501 E741 @@ -210,8 +211,8 @@ flake8-ignore = tests/test_pipeline_crawl.py E501 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py E501 - tests/test_pipeline_media.py E501 E741 - tests/test_proxy_connect.py E501 E741 + tests/test_pipeline_media.py E501 + tests/test_proxy_connect.py E501 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 @@ -222,8 +223,8 @@ flake8-ignore = tests/test_spidermiddleware_httperror.py E501 tests/test_spidermiddleware_offsite.py E501 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 E501 - tests/test_squeues.py E501 E741 + tests/test_spidermiddleware_referer.py E501 + tests/test_squeues.py E501 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 tests/test_utils_curl.py E501 @@ -232,12 +233,10 @@ flake8-ignore = tests/test_utils_deprecate.py E501 tests/test_utils_http.py E501 tests/test_utils_iterators.py E501 - tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 tests/test_utils_request.py E501 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 tests/test_utils_sitemap.py E501 tests/test_utils_url.py E501 E501 tests/test_webclient.py E501 E402 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 87d08cab7..35264a92b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,7 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "%s.%s".format( + return "{}.{}".format( f.__self__.__class__.__name__, f.__func__.__name__ ) diff --git a/scrapy/shell.py b/scrapy/shell.py index 08ce89481..2a3e13ddd 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -146,14 +146,13 @@ class Shell: b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " - "Fetch URL and update local objects " - "(by default, redirects are followed)") + "Fetch URL and update local objects (by default, redirects are followed)") b.append(" fetch(req) " "Fetch a scrapy.Request and update local objects ") b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % l for l in b) + return "\n".join("[s] %s" % line for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d368c7108..c5360bfa7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -96,5 +96,4 @@ def iterloc(it, alt=False): # Also consider alternate URLs (xhtml:link rel="alternate") if alt and 'alternate' in d: - for l in d['alternate']: - yield l + yield from d['alternate'] diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index c7a2ace88..133261fd7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''): def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython + @wraps(_embed_bpython_shell) def wrapper(namespace=namespace, banner=''): bpython.embed(locals_=namespace, banner=banner) @@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''): def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl + @wraps(_embed_ptpython_shell) def wrapper(namespace=namespace, banner=''): print(banner) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5207690f4..51d276097 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter): self.loggers = loggers or [] def filter(self, record): - if any(record.name.startswith(l + '.') for l in self.loggers): + if any(record.name.startswith(logger + '.') for logger in self.loggers): record.name = record.name.split('.', 1)[0] return True diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3d02d9478..9c1f3c2fe 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -152,6 +152,7 @@ def memoizemethod_noargs(method): weak reference to its object """ cache = weakref.WeakKeyDictionary() + @wraps(method) def new_method(self, *args, **kwargs): if self not in cache: diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 4ac7fb786..99c01c2b7 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase): def test_verbose_output(self): encoding = getattr(sys.stdout, 'encoding') or 'utf-8' _, out, _ = yield self.execute(['-v']) - headers = [l.partition(":")[0].strip() - for l in out.strip().decode(encoding).splitlines()] + headers = [ + line.partition(":")[0].strip() + for line in out.strip().decode(encoding).splitlines() + ] self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4215ca56c..84f80d103 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -104,44 +104,44 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_dns_error(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @@ -187,25 +187,25 @@ foo body with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) - self.assertEqual(str(l).count("Got response 200"), 1) + self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) def _assert_retried(self, log): self.assertEqual(str(log).count("Retrying"), 2) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index f86c50f50..d54434c8f 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check( + log.check( ('scrapy.downloadermiddlewares.cookies', 'DEBUG', 'Received cookies from: <200 http://scrapytest.org/>\n' @@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check() + log.check() def test_do_not_break_on_non_utf8_header(self): req = Request('http://scrapytest.org/') diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 38b95b710..41a8d16bc 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase): shutil.rmtree(path) def test_log(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present( + log.check_present( ( 'scrapy.dupefilters', 'DEBUG', @@ -188,7 +188,7 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close('finished') def test_log_debug(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -206,14 +206,14 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present( + log.check_present( ( 'scrapy.dupefilters', 'DEBUG', 'Filtered duplicate request: (referer: None)' ) ) - l.check_present( + log.check_present( ( 'scrapy.dupefilters', 'DEBUG', diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index b51f82422..19ff00350 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 1 - record = l.records[0] + assert len(log.records) == 1 + record = log.records[0] assert record.levelname == 'ERROR' self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 0 + assert len(log.records) == 0 @inlineCallbacks def test_default_process_item(self): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 188ec68dd..4763a5417 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2cf9135f..becacce62 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() @@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 25cd904bc..535f56691 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase): def test_top_level_logger(self): logger = logging.getLogger('test') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_children_logger(self): logger = logging.getLogger('test.test1') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_overlapping_name_logger(self): logger = logging.getLogger('test2') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test2', 'WARNING', 'test log msg')) + log.check(('test2', 'WARNING', 'test log msg')) def test_different_name_logger(self): logger = logging.getLogger('different') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('different', 'WARNING', 'test log msg')) + log.check(('different', 'WARNING', 'test log msg')) class LogCounterHandlerTest(unittest.TestCase): @@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase): sys.stdout = self.stdout def test_redirect(self): - with LogCapture() as l: + with LogCapture() as log: print('test log msg') - l.check(('test', 'ERROR', 'test log msg')) + log.check(('test', 'ERROR', 'test log msg')) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 015a0e5a2..28205e0d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c']) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index c83c9398c..b66588efb 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase): dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as l: + with LogCapture() as log: result = yield defer.maybeDeferred( self._get_result, test_signal, arg='test', handlers_called=handlers_called @@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) @@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase): test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as l: + with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(l.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(l)) + self.assertEqual(len(log.records), 1) + self.assertIn("Cannot return deferreds from signal handler", str(log)) dispatcher.disconnect(test_handler, test_signal)