RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into flake8-remove-e128

This commit is contained in:
Eugenio Lacuesta 2020-05-14 11:35:30 -03:00
parent e31b6ccc45 602bb13bdc
commit 89d0c98f7a
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
17 changed files with 90 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pytest.ini
View File

@ -176,13 +176,13 @@ flake8-ignore =
tests/test_command_shell.py E501
tests/test_commands.py E501
tests/test_contracts.py E501
tests/test_crawl.py E501 E741
tests/test_crawl.py E501
tests/test_crawler.py E501
tests/test_dependencies.py E501
tests/test_downloader_handlers.py E501
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E741 E501
tests/test_downloadermiddleware_cookies.py E501
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
@ -193,7 +193,7 @@ flake8-ignore =
tests/test_downloadermiddleware_retry.py E501
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741
tests/test_dupefilters.py E501
tests/test_engine.py E501
tests/test_exporters.py E501
tests/test_feedexport.py E501
@ -201,6 +201,7 @@ flake8-ignore =
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501
tests/test_http_response.py E501
tests/test_item.py E501
tests/test_link.py E501
tests/test_linkextractors.py E501
tests/test_loader.py E501 E741
@ -210,8 +211,8 @@ flake8-ignore =
tests/test_pipeline_crawl.py E501
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py E501
tests/test_pipeline_media.py E501 E741
tests/test_proxy_connect.py E501 E741
tests/test_pipeline_media.py E501
tests/test_proxy_connect.py E501
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
tests/test_robotstxt_interface.py E501 E501
@ -222,8 +223,8 @@ flake8-ignore =
tests/test_spidermiddleware_httperror.py E501
tests/test_spidermiddleware_offsite.py E501
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 E501
tests/test_squeues.py E501 E741
tests/test_spidermiddleware_referer.py E501
tests/test_squeues.py E501
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501
tests/test_utils_curl.py E501
@ -232,12 +233,10 @@ flake8-ignore =
tests/test_utils_deprecate.py E501
tests/test_utils_http.py E501
tests/test_utils_iterators.py E501
tests/test_utils_log.py E741
tests/test_utils_python.py E501
tests/test_utils_reqser.py E501
tests/test_utils_request.py E501
tests/test_utils_response.py E501
tests/test_utils_signal.py E741
tests/test_utils_sitemap.py E501
tests/test_utils_url.py E501 E501
tests/test_webclient.py E501 E402

2
scrapy/core/spidermw.py
View File

@ -19,7 +19,7 @@ def _isiterable(possible_iterator):
def _fname(f):
return "%s.%s".format(
return "{}.{}".format(
f.__self__.__class__.__name__,
f.__func__.__name__
)

5
scrapy/shell.py
View File

@ -146,14 +146,13 @@ class Shell:
b.append("Useful shortcuts:")
if self.inthread:
b.append(" fetch(url[, redirect=True]) "
"Fetch URL and update local objects "
"(by default, redirects are followed)")
"Fetch URL and update local objects (by default, redirects are followed)")
b.append(" fetch(req) "
"Fetch a scrapy.Request and update local objects ")
b.append(" shelp() Shell help (print this help)")
b.append(" view(response) View response in a browser")
return "\n".join("[s] %s" % l for l in b)
return "\n".join("[s] %s" % line for line in b)
def _is_relevant(self, value):
return isinstance(value, self.relevant_classes)

3
scrapy/spiders/sitemap.py
View File

@ -96,5 +96,4 @@ def iterloc(it, alt=False):
# Also consider alternate URLs (xhtml:link rel="alternate")
if alt and 'alternate' in d:
for l in d['alternate']:
yield l
yield from d['alternate']

2
scrapy/utils/console.py
View File

@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
def _embed_bpython_shell(namespace={}, banner=''):
"""Start a bpython shell"""
import bpython
@wraps(_embed_bpython_shell)
def wrapper(namespace=namespace, banner=''):
bpython.embed(locals_=namespace, banner=banner)
@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
def _embed_ptpython_shell(namespace={}, banner=''):
"""Start a ptpython shell"""
import ptpython.repl
@wraps(_embed_ptpython_shell)
def wrapper(namespace=namespace, banner=''):
print(banner)

2
scrapy/utils/log.py
View File

@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter):
self.loggers = loggers or []
def filter(self, record):
if any(record.name.startswith(l + '.') for l in self.loggers):
if any(record.name.startswith(logger + '.') for logger in self.loggers):
record.name = record.name.split('.', 1)[0]
return True

1
scrapy/utils/python.py
View File

@ -152,6 +152,7 @@ def memoizemethod_noargs(method):
weak reference to its object
"""
cache = weakref.WeakKeyDictionary()
@wraps(method)
def new_method(self, *args, **kwargs):
if self not in cache:

6
tests/test_command_version.py
View File

@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase):
def test_verbose_output(self):
encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
_, out, _ = yield self.execute(['-v'])
headers = [l.partition(":")[0].strip()
for l in out.strip().decode(encoding).splitlines()]
headers = [
line.partition(":")[0].strip()
for line in out.strip().decode(encoding).splitlines()
]
self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2',
'cssselect', 'parsel', 'w3lib',
'Twisted', 'Python', 'pyOpenSSL',

36
tests/test_crawl.py
View File

@ -104,44 +104,44 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_retry_503(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_failed(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_dns_error(self):
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
# try to fetch the homepage of a non-existent domain
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_start_requests_bug_before_yield(self):
with LogCapture('scrapy', level=logging.ERROR) as l:
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
self.assertEqual(len(l.records), 1)
record = l.records[0]
self.assertEqual(len(log.records), 1)
record = log.records[0]
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@defer.inlineCallbacks
def test_start_requests_bug_yielding(self):
with LogCapture('scrapy', level=logging.ERROR) as l:
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
self.assertEqual(len(l.records), 1)
record = l.records[0]
self.assertEqual(len(log.records), 1)
record = log.records[0]
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@ -187,25 +187,25 @@ foo body
with multiples lines
'''})
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
self.assertEqual(str(l).count("Got response 200"), 1)
self.assertEqual(str(log).count("Got response 200"), 1)
@defer.inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
crawler = self.runner.create_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
self._assert_retried(l)
self._assert_retried(log)
def _assert_retried(self, log):
self.assertEqual(str(log).count("Retrying"), 2)

8
tests/test_downloadermiddleware_cookies.py
View File

@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase):
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as l:
level=logging.DEBUG) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
l.check(
log.check(
('scrapy.downloadermiddlewares.cookies',
'DEBUG',
'Received cookies from: <200 http://scrapytest.org/>\n'
@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase):
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as l:
level=logging.DEBUG) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
l.check()
log.check()
def test_do_not_break_on_non_utf8_header(self):
req = Request('http://scrapytest.org/')

10
tests/test_dupefilters.py
View File

@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase):
shutil.rmtree(path)
def test_log(self):
with LogCapture() as l:
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',
@ -188,7 +188,7 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.close('finished')
def test_log_debug(self):
with LogCapture() as l:
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
@ -206,14 +206,14 @@ class RFPDupeFilterTest(unittest.TestCase):
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
)
)
l.check_present(
log.check_present(
(
'scrapy.dupefilters',
'DEBUG',

10
tests/test_pipeline_media.py
View File

@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
fail = Failure(Exception())
results = [(True, 1), (False, fail)]
with LogCapture() as l:
with LogCapture() as log:
new_item = self.pipe.item_completed(results, item, self.info)
assert new_item is item
assert len(l.records) == 1
record = l.records[0]
assert len(log.records) == 1
record = log.records[0]
assert record.levelname == 'ERROR'
self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
# disable failure logging and check again
self.pipe.LOG_FAILED_RESULTS = False
with LogCapture() as l:
with LogCapture() as log:
new_item = self.pipe.item_completed(results, item, self.info)
assert new_item is item
assert len(l.records) == 0
assert len(log.records) == 0
@inlineCallbacks
def test_default_process_item(self):

16
tests/test_proxy_connect.py
View File

@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase):
@defer.inlineCallbacks
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, log)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl("https://localhost:99999/status?n=200")
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(log)
@defer.inlineCallbacks
def test_https_tunnel_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
crawler = get_crawler(SimpleSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
# The proxy returns a 407 error code but it does not reach the client;
# he just sees a TunnelError.
self._assert_got_tunnel_error(l)
self._assert_got_tunnel_error(log)
@defer.inlineCallbacks
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)
with LogCapture() as l:
with LogCapture() as log:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, l)
self._assert_got_response_code(200, log)
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])

24
tests/test_squeues.py
View File

@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
def test_serialize_loader(self):
q = self.queue()
l = TestLoader()
q.push(l)
l2 = q.pop()
assert isinstance(l2, TestLoader)
assert l2.default_item_class is TestItem
self.assertEqual(l2.name_out('x'), 'xx')
loader = TestLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
self.assertEqual(loader2.name_out('x'), 'xx')
def test_serialize_request_recursive(self):
q = self.queue()
@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def test_serialize_loader(self):
q = self.queue()
l = TestLoader()
q.push(l)
l2 = q.pop()
assert isinstance(l2, TestLoader)
assert l2.default_item_class is TestItem
self.assertEqual(l2.name_out('x'), 'xx')
loader = TestLoader()
q.push(loader)
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
self.assertEqual(loader2.name_out('x'), 'xx')
def test_serialize_request_recursive(self):
q = self.queue()

24
tests/test_utils_log.py
View File

@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase):
def test_top_level_logger(self):
logger = logging.getLogger('test')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test', 'WARNING', 'test log msg'))
log.check(('test', 'WARNING', 'test log msg'))
def test_children_logger(self):
logger = logging.getLogger('test.test1')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test', 'WARNING', 'test log msg'))
log.check(('test', 'WARNING', 'test log msg'))
def test_overlapping_name_logger(self):
logger = logging.getLogger('test2')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('test2', 'WARNING', 'test log msg'))
log.check(('test2', 'WARNING', 'test log msg'))
def test_different_name_logger(self):
logger = logging.getLogger('different')
with self.handler as l:
with self.handler as log:
logger.warning('test log msg')
l.check(('different', 'WARNING', 'test log msg'))
log.check(('different', 'WARNING', 'test log msg'))
class LogCounterHandlerTest(unittest.TestCase):
@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase):
sys.stdout = self.stdout
def test_redirect(self):
with LogCapture() as l:
with LogCapture() as log:
print('test log msg')
l.check(('test', 'ERROR', 'test log msg'))
log.check(('test', 'ERROR', 'test log msg'))

4
tests/test_utils_misc/__init__.py
View File

@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase):
assert hasattr(arg_to_iter(100), '__iter__')
assert hasattr(arg_to_iter('lala'), '__iter__')
assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')
self.assertEqual(list(arg_to_iter(None)), [])
self.assertEqual(list(arg_to_iter('lala')), ['lala'])
self.assertEqual(list(arg_to_iter(100)), [100])
self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])
self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])

12
tests/test_utils_signal.py
View File

@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase):
dispatcher.connect(self.error_handler, signal=test_signal)
dispatcher.connect(self.ok_handler, signal=test_signal)
with LogCapture() as l:
with LogCapture() as log:
result = yield defer.maybeDeferred(
self._get_result, test_signal, arg='test',
handlers_called=handlers_called
@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase):
assert self.error_handler in handlers_called
assert self.ok_handler in handlers_called
self.assertEqual(len(l.records), 1)
record = l.records[0]
self.assertEqual(len(log.records), 1)
record = log.records[0]
self.assertIn('error_handler', record.getMessage())
self.assertEqual(record.levelname, 'ERROR')
self.assertEqual(result[0][0], self.error_handler)
@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase):
test_signal = object()
dispatcher.connect(test_handler, test_signal)
with LogCapture() as l:
with LogCapture() as log:
send_catch_log(test_signal)
self.assertEqual(len(l.records), 1)
self.assertIn("Cannot return deferreds from signal handler", str(l))
self.assertEqual(len(log.records), 1)
self.assertIn("Cannot return deferreds from signal handler", str(log))
dispatcher.disconnect(test_handler, test_signal)