RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Keep 2 spaces between code and inline comments (#4195)

This commit is contained in:
Adrián Chaves 2019-12-13 14:20:48 +01:00 committed by GitHub
parent 02cdc53fb8
commit b5c4c2cae8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 54 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pytest.ini
View File

@ -26,8 +26,8 @@ flake8-ignore =
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E261 E501
extras/qpsclient.py E501 E261 E501
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
@ -45,15 +45,15 @@ flake8-ignore =
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E502 E128
# scrapy/core
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E306 E261 E128 W504
scrapy/core/scraper.py E501 E306 E128 W504
scrapy/core/spidermw.py E501 E731 E502 E126 E226
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
scrapy/core/downloader/handlers/http10.py E501
@ -102,7 +102,7 @@ flake8-ignore =
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
@ -112,17 +112,16 @@ flake8-ignore =
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501 E261
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E502 E501
scrapy/utils/console.py E261 E306 E305
scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/engine.py E261
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
@ -150,7 +149,7 @@ flake8-ignore =
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501 E261 E226
scrapy/exporters.py E501 E226
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
@ -171,7 +170,7 @@ flake8-ignore =
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_fetch.py E501 E261
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
@ -179,7 +178,7 @@ flake8-ignore =
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@ -194,13 +193,13 @@ flake8-ignore =
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
tests/test_engine.py E401 E501 E502 E128 E261
tests/test_engine.py E401 E501 E502 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123
tests/test_http_response.py E501 E301 E502 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
@ -212,7 +211,7 @@ flake8-ignore =
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py E501 W293 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
@ -224,12 +223,12 @@ flake8-ignore =
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
tests/test_spidermiddleware_output_chain.py E501 W293 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
tests/test_utils_defer.py E306 E261 E501 F841 E226
tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
tests/test_utils_http.py E501 E502 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
@ -240,7 +239,7 @@ flake8-ignore =
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_spider.py E261 E305
tests/test_utils_spider.py E305
tests/test_utils_template.py E305
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126

2
scrapy/core/downloader/webclient.py
View File

@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
self.headers = Headers() # bucket for response headers
self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)

2
scrapy/core/engine.py
View File

@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
self.inprogress = set() # requests in progress
self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall

2
scrapy/core/scraper.py
View File

@ -123,7 +123,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd

2
scrapy/exporters.py
View File

@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
)
self.csv_writer = csv.writer(self.stream, **kwargs)
self._headers_not_written = True

4
scrapy/settings/default_settings.py
View File

@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
# also allowing negotiation
# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}

4
scrapy/spiders/feed.py
View File

@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
and the file's headers.
"""
delimiter = None # When this is None, python's csv module's default delimiter is used
quotechar = None # When this is None, python's csv module's default quotechar is used
delimiter = None # When this is None, python's csv module's default delimiter is used
quotechar = None # When this is None, python's csv module's default quotechar is used
headers = None
def process_results(self, response, results):

8
scrapy/utils/console.py
View File

@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
def _embed_standard_shell(namespace={}, banner=''):
"""Start a standard python shell"""
import code
try: # readline module is only available on unix systems
try: # readline module is only available on unix systems
import readline
except ImportError:
pass
@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):
"""Return the first acceptable shell-embed function
from a given list of shell names.
"""
if shells is None: # list, preference order of shells
if shells is None: # list, preference order of shells
shells = DEFAULT_PYTHON_SHELLS.keys()
if known_shells is None: # available embeddable shells
if known_shells is None: # available embeddable shells
known_shells = DEFAULT_PYTHON_SHELLS.copy()
for shell in shells:
if shell in known_shells:
@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
shell = get_shell_embed_func(shells)
if shell is not None:
shell(namespace=namespace, banner=banner)
except SystemExit: # raised when using exit() in python code.interact
except SystemExit: # raised when using exit() in python code.interact
pass

2
tests/test_command_fetch.py
View File

@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
out = out.replace(b'\r', b'') # required on win32
out = out.replace(b'\r', b'') # required on win32
assert b'Server: TwistedWeb' in out, out
assert b'Content-Type: text/plain' in out

2
tests/test_downloader_handlers.py
View File

@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase):
return self.download_request(request, Spider('foo')).addCallback(_test)
def test_payload(self):
body = b'1'*100 # PayloadResource requires body length to be 100
body = b'1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
d = self.download_request(request, Spider('foo'))
d.addCallback(lambda r: r.body)

1
tests/test_engine.py
View File

@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase):
self.run.signals_catched[signals.spider_opened])
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_idle])
self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
self.run.signals_catched[signals.spider_closed])

2
tests/test_http_request.py
View File

@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase):
r2 = self.request_class(url="http://www.example.com/", body=b"")
assert isinstance(r2.body, bytes)
self.assertEqual(r2.encoding, 'utf-8') # default encoding
self.assertEqual(r2.encoding, 'utf-8') # default encoding
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
assert isinstance(r3.body, bytes)

6
tests/test_pipeline_media.py
View File

@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
m = self.pipe._mockcalled
# only once
self.assertEqual(m[0], 'get_media_requests') # first hook called
self.assertEqual(m[0], 'get_media_requests') # first hook called
self.assertEqual(m.count('get_media_requests'), 1)
self.assertEqual(m.count('item_completed'), 1)
self.assertEqual(m[-1], 'item_completed') # last hook called
self.assertEqual(m[-1], 'item_completed') # last hook called
# twice, one per request
self.assertEqual(m.count('media_to_download'), 2)
# one to handle success and other for failure
@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def test_get_media_requests(self):
# returns single Request (without callback)
req = Request('http://url')
item = dict(requests=req) # pass a single item
item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
assert request_fingerprint(req) in self.info.downloaded

20
tests/test_spidermiddleware_referer.py
View File

@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
b'http://scrapytest.org/1', # expected initial referer
b'http://scrapytest.org/1', # expected referer for the redirection request
b'http://scrapytest.org/1', # expected initial referer
b'http://scrapytest.org/1', # expected referer for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
None, # expected initial "Referer"
None, # expected "Referer" for the redirection request
None, # expected initial "Referer"
None, # expected "Referer" for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
b'http://scrapytest.org/101', # expected initial "Referer"
b'http://scrapytest.org/101', # expected referer for the redirection request
b'http://scrapytest.org/101', # expected initial "Referer"
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',

8
tests/test_utils_defer.py
View File

@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):
return steps
dfd = mustbe_deferred(_append, 1)
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
steps.append(2) # add another value, that should be catched by assertEqual
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
steps.append(2) # add another value, that should be catched by assertEqual
return dfd
def test_unfired_deferred(self):
@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase):
return dfd
dfd = mustbe_deferred(_append, 1)
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
steps.append(2) # add another value, that should be catched by assertEqual
dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
steps.append(2) # add another value, that should be catched by assertEqual
return dfd

13
tests/test_utils_spider.py
View File

@ -1,20 +1,16 @@
import unittest
from scrapy import Spider
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
from scrapy.spiders import CrawlSpider
class MyBaseSpider(CrawlSpider):
pass # abstract spider
class MySpider1(MyBaseSpider):
class MySpider1(Spider):
name = 'myspider1'
class MySpider2(MyBaseSpider):
class MySpider2(Spider):
name = 'myspider2'
@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):
it = iter_spider_classes(tests.test_utils_spider)
self.assertEqual(set(it), {MySpider1, MySpider2})
if __name__ == "__main__":
unittest.main()