diff --git a/pytest.ini b/pytest.ini index 33c34b8e8..1b23595c0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -26,8 +26,8 @@ flake8-ignore = scrapy/http/__init__.py F401 # Issues pending a review: # extras - extras/qps-bench-server.py E261 E501 - extras/qpsclient.py E501 E261 E501 + extras/qps-bench-server.py E501 + extras/qpsclient.py E501 E501 # scrapy/commands scrapy/commands/__init__.py E128 E501 scrapy/commands/check.py E501 @@ -45,15 +45,15 @@ flake8-ignore = scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E502 E128 # scrapy/core - scrapy/core/engine.py E261 E501 E128 E127 E306 E502 + scrapy/core/engine.py E501 E128 E127 E306 E502 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E306 E261 E128 W504 + scrapy/core/scraper.py E501 E306 E128 W504 scrapy/core/spidermw.py E501 E731 E502 E126 E226 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 - scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226 + scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 scrapy/core/downloader/handlers/http10.py E501 @@ -102,7 +102,7 @@ flake8-ignore = scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E261 E114 E116 E226 + scrapy/settings/default_settings.py E501 E114 E116 E226 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -112,17 +112,16 @@ flake8-ignore = # scrapy/spiders scrapy/spiders/__init__.py E501 E402 scrapy/spiders/crawl.py E501 - scrapy/spiders/feed.py E501 E261 + scrapy/spiders/feed.py E501 scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E261 E306 E305 + scrapy/utils/console.py E306 E305 scrapy/utils/datatypes.py E501 E226 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/engine.py E261 scrapy/utils/gz.py E305 E501 W504 scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 @@ -150,7 +149,7 @@ flake8-ignore = scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 scrapy/exceptions.py E501 - scrapy/exporters.py E501 E261 E226 + scrapy/exporters.py E501 E226 scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 @@ -171,7 +170,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 - tests/test_command_fetch.py E501 E261 + tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 @@ -179,7 +178,7 @@ flake8-ignore = tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 @@ -194,13 +193,13 @@ flake8-ignore = tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 - tests/test_engine.py E401 E501 E502 E128 E261 + tests/test_engine.py E401 E501 E502 E128 tests/test_exporters.py E501 E731 E306 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 + tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 @@ -212,7 +211,7 @@ flake8-ignore = tests/test_pipeline_crawl.py E131 E501 E128 E126 tests/test_pipeline_files.py E501 W293 E303 E272 E226 tests/test_pipeline_images.py F841 E501 E303 - tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 + tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 E305 @@ -224,12 +223,12 @@ flake8-ignore = tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 tests/test_spidermiddleware_output_chain.py E501 W293 E226 - tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121 + tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E261 E501 F841 E226 + tests/test_utils_defer.py E306 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 tests/test_utils_http.py E501 E502 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E303 E241 @@ -240,7 +239,7 @@ flake8-ignore = tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 E731 E226 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_spider.py E261 E305 + tests/test_utils_spider.py E305 tests/test_utils_template.py E305 tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 798346f19..f368c3bae 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient): delimiter = b'\n' def connectionMade(self): - self.headers = Headers() # bucket for response headers + self.headers = Headers() # bucket for response headers # Method command self.sendCommand(self.factory.method, self.factory.path) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fa913e528..829e69993 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -25,7 +25,7 @@ class Slot(object): def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False - self.inprogress = set() # requests in progress + self.inprogress = set() # requests in progress self.start_requests = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index db463f989..b3d585cce 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -123,7 +123,7 @@ class Scraper(object): callback/errback""" assert isinstance(response, (Response, Failure)) - dfd = self._scrape2(response, request, spider) # returns spiders processed output + dfd = self._scrape2(response, request, spider) # returns spiders processed output dfd.addErrback(self.handle_spider_error, request, response, spider) dfd.addCallback(self.handle_spider_output, request, response, spider) return dfd diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fcb55da67..5bf131312 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5c9678c01..1e163e1fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT' -DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, - # also allowing negotiation +# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: +DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 197812a26..c566f0236 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -100,8 +100,8 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = None # When this is None, python's csv module's default delimiter is used - quotechar = None # When this is None, python's csv module's default quotechar is used + delimiter = None # When this is None, python's csv module's default delimiter is used + quotechar = None # When this is None, python's csv module's default quotechar is used headers = None def process_results(self, response, results): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 688e28c34..7eb40f0ce 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''): def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code - try: # readline module is only available on unix systems + try: # readline module is only available on unix systems import readline except ImportError: pass @@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None): """Return the first acceptable shell-embed function from a given list of shell names. """ - if shells is None: # list, preference order of shells + if shells is None: # list, preference order of shells shells = DEFAULT_PYTHON_SHELLS.keys() - if known_shells is None: # available embeddable shells + if known_shells is None: # available embeddable shells known_shells = DEFAULT_PYTHON_SHELLS.copy() for shell in shells: if shell in known_shells: @@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None): shell = get_shell_embed_func(shells) if shell is not None: shell(namespace=namespace, banner=banner) - except SystemExit: # raised when using exit() in python code.interact + except SystemExit: # raised when using exit() in python code.interact pass diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 3fa3ed930..9d3c8fe73 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace(b'\r', b'') # required on win32 + out = out.replace(b'\r', b'') # required on win32 assert b'Server: TwistedWeb' in out, out assert b'Content-Type: text/plain' in out diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 60124b93f..2db2417e8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_payload(self): - body = b'1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) diff --git a/tests/test_engine.py b/tests/test_engine.py index 002c4e6bb..537df8d91 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase): self.run.signals_catched[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, self.run.signals_catched[signals.spider_idle]) - self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17 self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, self.run.signals_catched[signals.spider_closed]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a98aa1e6f..57e7b457d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, 'utf-8') # default encoding + self.assertEqual(r2.encoding, 'utf-8') # default encoding r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0d23f51cc..1fcc5799e 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled # only once - self.assertEqual(m[0], 'get_media_requests') # first hook called + self.assertEqual(m[0], 'get_media_requests') # first hook called self.assertEqual(m.count('get_media_requests'), 1) self.assertEqual(m.count('item_completed'), 1) - self.assertEqual(m[-1], 'item_completed') # last hook called + self.assertEqual(m[-1], 'item_completed') # last hook called # twice, one per request self.assertEqual(m.count('media_to_download'), 2) # one to handle success and other for failure @@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request('http://url') - item = dict(requests=req) # pass a single item + item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert request_fingerprint(req) in self.info.downloaded diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index ecec6135d..7cc17600c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - b'http://scrapytest.org/1', # expected initial referer - b'http://scrapytest.org/1', # expected referer for the redirection request + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - None, # expected initial "Referer" - None, # expected "Referer" for the redirection request + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial "Referer" - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 49c2befb5..dfbe71ae2 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd def test_unfired_deferred(self): @@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index edeeacc80..ee7d17062 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,20 +1,16 @@ import unittest + +from scrapy import Spider from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils.spider import iterate_spider_output, iter_spider_classes -from scrapy.spiders import CrawlSpider - -class MyBaseSpider(CrawlSpider): - pass # abstract spider - - -class MySpider1(MyBaseSpider): +class MySpider1(Spider): name = 'myspider1' -class MySpider2(MyBaseSpider): +class MySpider2(Spider): name = 'myspider2' @@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase): it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) + if __name__ == "__main__": unittest.main()