diff --git a/pytest.ini b/pytest.ini index a66c08652..8177fedd3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,52 +31,51 @@ flake8-ignore = extras/qps-bench-server.py E501 extras/qpsclient.py E501 E501 # scrapy/commands - scrapy/commands/__init__.py E128 E501 + scrapy/commands/__init__.py E501 scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E501 E128 - scrapy/commands/genspider.py E128 E501 - scrapy/commands/parse.py E128 E501 + scrapy/commands/fetch.py E501 + scrapy/commands/genspider.py E501 + scrapy/commands/parse.py E501 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E128 - scrapy/commands/shell.py E128 E501 - scrapy/commands/startproject.py E501 E128 - scrapy/commands/version.py E501 E128 + scrapy/commands/settings.py E501 + scrapy/commands/shell.py E501 + scrapy/commands/startproject.py E501 + scrapy/commands/version.py E501 # scrapy/contracts scrapy/contracts/__init__.py E501 - scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 + scrapy/core/engine.py E501 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 + scrapy/core/scraper.py E501 scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 + scrapy/core/downloader/contextfactory.py E501 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 + scrapy/core/downloader/webclient.py E501 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 + scrapy/core/downloader/handlers/ftp.py E501 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 + scrapy/core/downloader/handlers/s3.py E501 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 - scrapy/downloadermiddlewares/httpcompression.py E501 E128 + scrapy/downloadermiddlewares/httpcompression.py E501 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 scrapy/downloadermiddlewares/retry.py E501 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 + scrapy/extensions/closespider.py E501 scrapy/extensions/corestats.py E501 - scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 + scrapy/extensions/feedexport.py E501 + scrapy/extensions/httpcache.py E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 @@ -87,13 +86,13 @@ flake8-ignore = scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 + scrapy/http/response/__init__.py E501 + scrapy/http/response/text.py E501 # scrapy/linkextractors scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader - scrapy/loader/__init__.py E501 E128 + scrapy/loader/__init__.py E501 scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 @@ -123,13 +122,13 @@ flake8-ignore = scrapy/utils/conf.py E402 E501 scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 - scrapy/utils/defer.py E501 E128 + scrapy/utils/defer.py E501 scrapy/utils/deprecate.py E501 scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 - scrapy/utils/log.py E128 E501 + scrapy/utils/log.py E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 @@ -138,13 +137,13 @@ flake8-ignore = scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E501 - scrapy/utils/response.py E501 E128 - scrapy/utils/signal.py E501 E128 + scrapy/utils/response.py E501 + scrapy/utils/signal.py E501 scrapy/utils/sitemap.py E501 scrapy/utils/spider.py E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 E128 F405 + scrapy/utils/url.py E501 F403 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/cmdline.py E501 @@ -156,16 +155,15 @@ flake8-ignore = scrapy/item.py E501 scrapy/link.py E501 scrapy/logformatter.py E501 - scrapy/mail.py E402 E128 E501 - scrapy/middleware.py E128 E501 + scrapy/mail.py E402 E501 + scrapy/middleware.py E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 + scrapy/responsetypes.py E501 scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 scrapy/spiderloader.py E501 - scrapy/squeues.py E128 scrapy/squeues.py E501 scrapy/statscollectors.py E501 # tests @@ -174,47 +172,47 @@ flake8-ignore = tests/spiders.py E501 tests/test_closespider.py E501 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 - tests/test_command_shell.py E501 E128 - tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 - tests/test_crawl.py E501 E741 + tests/test_command_parse.py E501 + tests/test_command_shell.py E501 + tests/test_commands.py E501 + tests/test_contracts.py E501 + tests/test_crawl.py E501 tests/test_crawler.py E501 tests/test_dependencies.py E501 - tests/test_downloader_handlers.py E128 E501 + tests/test_downloader_handlers.py E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 + tests/test_downloadermiddleware_cookies.py E501 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 tests/test_downloadermiddleware_httpcompression.py E501 tests/test_downloadermiddleware_decompression.py E501 - tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 - tests/test_downloadermiddleware_retry.py E501 E128 + tests/test_downloadermiddleware_httpproxy.py E501 + tests/test_downloadermiddleware_redirect.py E501 + tests/test_downloadermiddleware_retry.py E501 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 - tests/test_engine.py E501 E128 - tests/test_exporters.py E501 E128 + tests/test_dupefilters.py E501 + tests/test_engine.py E501 + tests/test_exporters.py E501 tests/test_feedexport.py E501 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E128 E128 - tests/test_http_response.py E501 E128 - tests/test_item.py E128 E501 + tests/test_http_request.py E402 E501 + tests/test_http_response.py E501 + tests/test_item.py E501 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 - tests/test_logformatter.py E128 E501 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 + tests/test_linkextractors.py E501 + tests/test_loader.py E501 E741 + tests/test_logformatter.py E501 + tests/test_mail.py E501 + tests/test_middleware.py E501 + tests/test_pipeline_crawl.py E501 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py E501 - tests/test_pipeline_media.py E501 E741 E128 - tests/test_proxy_connect.py E501 E741 + tests/test_pipeline_media.py E501 + tests/test_proxy_connect.py E501 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 @@ -222,29 +220,27 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 - tests/test_spidermiddleware_offsite.py E501 E128 + tests/test_spidermiddleware_httperror.py E501 + tests/test_spidermiddleware_offsite.py E501 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 E501 - tests/test_squeues.py E501 E741 + tests/test_spidermiddleware_referer.py E501 + tests/test_squeues.py E501 tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E128 + tests/test_utils_conf.py E501 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 tests/test_utils_defer.py E501 tests/test_utils_deprecate.py E501 - tests/test_utils_http.py E501 E128 - tests/test_utils_iterators.py E501 E128 - tests/test_utils_log.py E741 + tests/test_utils_http.py E501 + tests/test_utils_iterators.py E501 tests/test_utils_python.py E501 - tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 + tests/test_utils_reqser.py E501 + tests/test_utils_request.py E501 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 - tests/test_utils_sitemap.py E128 E501 + tests/test_utils_sitemap.py E501 tests/test_utils_url.py E501 E501 - tests/test_webclient.py E501 E128 E402 + tests/test_webclient.py E501 E402 tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 E128 - tests/test_spiderloader/__init__.py E128 E501 + tests/test_settings/__init__.py E501 + tests/test_spiderloader/__init__.py E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9f8e6986a..97f818cb2 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -59,17 +59,17 @@ class ScrapyCommand: """ group = OptionGroup(parser, "Global Options") group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") + help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help="log level (default: %s)" % self.settings['LOG_LEVEL']) group.add_option("--nolog", action="store_true", - help="disable logging completely") + help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") + help="write python cProfile stats to FILE") group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") + help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") + help="set/override setting (may be repeated)") group.add_option("--pdb", action="store_true", help="enable pdb on failure") parser.add_option_group(group) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 506d1f1b7..063195f50 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,8 +27,8 @@ class Command(ScrapyCommand): parser.add_option("--spider", dest="spider", help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2e837abed..abf3b7a5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -36,15 +36,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") + help="List available templates") parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") + help="Edit spider after creating it") parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") + help="Dump template to standard output") parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") + help="Uses a custom template.") parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 098827ab9..580fd2828 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -33,29 +33,29 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") + help="use this spider without looking for one") parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") + help="set spider argument (may be repeated)") parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") + help="process items through pipelines") parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") + help="don't show links to follow (extracted requests)") parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") + help="don't show scraped items") parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") + help="avoid using pygments to colorize the output") parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") + help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") + help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") + help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") + help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 603bafb9f..8d49e440f 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -19,15 +19,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") + help="print raw setting value") parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") + help="print setting value, interpreted as a boolean") parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") + help="print setting value, interpreted as an integer") parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") + help="print setting value, interpreted as a float") parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 5946f21e8..d1944df3d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -34,11 +34,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") + help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + help="use this spider") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index b123e5c84..ebe3a9c2c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -102,10 +102,8 @@ class Command(ScrapyCommand): move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) - tplfile = join(project_dir, - string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, - ProjectName=string_camelcase(project_name)) + tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) + render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) print("New Scrapy project '%s', using template directory '%s', " "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1516c5997..d0ea72a67 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -17,7 +17,7 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ab73e12c8..452242d47 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # # * getattr() for `_ssl_method` attribute for context factories # not calling super(..., self).__init__ - return CertificateOptions(verify=False, - method=getattr(self, 'method', - getattr(self, '_ssl_method', None)), - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers) + return CertificateOptions( + verify=False, + method=getattr(self, 'method', getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=self.tls_ciphers, + ) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 94b55c347..3ef129587 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -86,10 +86,9 @@ class FTPDownloadHandler: password = request.meta.get("ftp_password", self.default_password) passive_mode = 1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0 - creator = ClientCreator(reactor, FTPClient, user, password, - passive=passive_mode) - return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, - request, unquote(parsed_url.path)) + creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode) + dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) def gotClient(self, client, request, filepath): self.client = client diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 324d21716..de0da4b70 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -217,11 +217,9 @@ class ExecutionEngine: self.slot.nextcall.schedule() def schedule(self, request, spider): - self.signals.send_catch_log(signal=signals.request_scheduled, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) if not self.slot.scheduler.enqueue_request(request): - self.signals.send_catch_log(signal=signals.request_dropped, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) def download(self, request, spider): d = self._download(request, spider) @@ -247,8 +245,8 @@ class ExecutionEngine: logkws = self.logformatter.crawled(request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signal=signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log(signals.response_received, + response=response, request=request, spider=spider) return response def _on_complete(_): @@ -286,8 +284,7 @@ class ExecutionEngine: next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = self.signals.send_catch_log(signal=signals.spider_idle, - spider=spider, dont_log=DontCloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): return diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 87d08cab7..35264a92b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,7 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "%s.%s".format( + return "{}.{}".format( f.__self__.__class__.__name__, f.__func__.__name__ ) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7972b58b1..6294a9b52 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -46,9 +46,10 @@ class RFC2616Policy: def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = [to_bytes(cc) for cc in - settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')] self._cc_parsed = WeakKeyDictionary() + self.ignore_response_cache_controls = [ + to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + ] def _parse_cachecontrol(self, r): if r not in self._cc_parsed: diff --git a/scrapy/mail.py b/scrapy/mail.py index 9d7896ef6..7d7a2c435 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -28,8 +28,10 @@ def _to_bytes_or_none(text): class MailSender: - def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', - smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): + def __init__( + self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, + smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False + ): self.smtphost = smtphost self.smtpport = smtpport self.smtpuser = _to_bytes_or_none(smtpuser) @@ -41,9 +43,15 @@ class MailSender: @classmethod def from_settings(cls, settings): - return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'], - settings['MAIL_PASS'], settings.getint('MAIL_PORT'), - settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) + return cls( + smtphost=settings['MAIL_HOST'], + mailfrom=settings['MAIL_FROM'], + smtpuser=settings['MAIL_USER'], + smtppass=settings['MAIL_PASS'], + smtpport=settings.getint('MAIL_PORT'), + smtptls=settings.getbool('MAIL_TLS'), + smtpssl=settings.getbool('MAIL_SSL'), + ) def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None): from twisted.internet import reactor @@ -89,9 +97,12 @@ class MailSender: return dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) - dfd.addCallbacks(self._sent_ok, self._sent_failed, + dfd.addCallbacks( + callback=self._sent_ok, + errback=self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)]) + errbackArgs=[to, cc, subject, len(attachs)], + ) reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) return dfd @@ -115,9 +126,10 @@ class MailSender: from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, - to_addrs, msg, d, heloFallback=True, requireAuthentication=False, - requireTransportSecurity=self.smtptls) + factory = ESMTPSenderFactory( + self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, + heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, + ) factory.noisy = False if self.smtpssl: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7c5eeac21..d207088e6 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -58,9 +58,9 @@ class ResponseTypes: def from_content_disposition(self, content_disposition): try: - filename = to_unicode(content_disposition, - encoding='latin-1', errors='replace').split(';')[1].split('=')[1] - filename = filename.strip('"\'') + filename = to_unicode( + content_disposition, encoding='latin-1', errors='replace' + ).split(';')[1].split('=')[1].strip('"\'') return self.from_filename(filename) except IndexError: return Response diff --git a/scrapy/shell.py b/scrapy/shell.py index 83afb74c9..3ff5a8ad8 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -145,14 +145,13 @@ class Shell: b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " - "Fetch URL and update local objects " - "(by default, redirects are followed)") + "Fetch URL and update local objects (by default, redirects are followed)") b.append(" fetch(req) " "Fetch a scrapy.Request and update local objects ") b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % l for l in b) + return "\n".join("[s] %s" % line for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d368c7108..c5360bfa7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -96,5 +96,4 @@ def iterloc(it, alt=False): # Also consider alternate URLs (xhtml:link rel="alternate") if alt and 'alternate' in d: - for l in d['alternate']: - yield l + yield from d['alternate'] diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index c7a2ace88..133261fd7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''): def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython + @wraps(_embed_bpython_shell) def wrapper(namespace=namespace, banner=''): bpython.embed(locals_=namespace, banner=banner) @@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''): def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl + @wraps(_embed_ptpython_shell) def wrapper(namespace=namespace, banner=''): print(banner) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 34b8d9774..a3950db75 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw, - errbackArgs=a, errbackKeywords=kw) + d.addCallbacks( + callback=cb, errback=eb, + callbackArgs=a, callbackKeywords=kw, + errbackArgs=a, errbackKeywords=kw, + ) if isinstance(input, failure.Failure): d.errback(input) else: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 203842fc8..51d276097 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter): self.loggers = loggers or [] def filter(self, record): - if any(record.name.startswith(l + '.') for l in self.loggers): + if any(record.name.startswith(logger + '.') for logger in self.loggers): record.name = record.name.split('.', 1)[0] return True @@ -142,10 +142,12 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Versions: %(versions)s", - {'versions': ", ".join("%s %s" % (name, version) - for name, version in scrapy_components_versions() - if name != "Scrapy")}) + versions = [ + "%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy" + ] + logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3d02d9478..9c1f3c2fe 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -152,6 +152,7 @@ def memoizemethod_noargs(method): weak reference to its object """ cache = weakref.WeakKeyDictionary() + @wraps(method) def new_method(self, *args, **kwargs): if self not in cache: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 29fdaaf2c..edbc0db25 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -19,8 +19,7 @@ def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] - _baseurl_cache[response] = html.get_base_url(text, response.url, - response.encoding) + _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh(text, response.url, - response.encoding, ignore_tags=ignore_tags) + _metaref_cache[response] = html.get_meta_refresh( + text, response.url, response.encoding, ignore_tags=ignore_tags) return _metaref_cache[response] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 60c561da6..a311e9257 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): try: - response = robustApply(receiver, signal=signal, sender=sender, - *arguments, **named) + response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named) if isinstance(response, Deferred): logger.error("Cannot return deferreds from signal handler: %(receiver)s", {'receiver': receiver}, extra={'spider': spider}) @@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, - *arguments, **named) + d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) dfds.append(d) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..955b63d4b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, - [spider.name] + list(getattr(spider, 'allowed_domains', []))) + return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', []))) def url_has_any_extension(url, extensions): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 85a24d0bc..a09dcf072 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} @defer.inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-c', 'parse_request_without_meta', - '--nolinks', + '-c', 'parse_request_without_meta', + '--nolinks', self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..93fda2648 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_nofile(self): filepath = 'file:///tests/sample_data/test_site/nothinghere.html' - errcode, out, err = yield self.execute([filepath, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'No such file or directory', err) @defer.inlineCallbacks def test_dns_failures(self): url = 'www.somedomainthatdoesntexi.st' - errcode, out, err = yield self.execute([url, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 4ac7fb786..99c01c2b7 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase): def test_verbose_output(self): encoding = getattr(sys.stdout, 'encoding') or 'utf-8' _, out, _ = yield self.execute(['-v']) - headers = [l.partition(":")[0].strip() - for l in out.strip().decode(encoding).splitlines()] + headers = [ + line.partition(":")[0].strip() + for line in out.strip().decode(encoding).splitlines() + ] self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d1ce80f9d..99120b128 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) self.assertEqual(len(contracts), 2) - self.assertEqual(frozenset(type(x) for x in contracts), + self.assertEqual( + frozenset(type(x) for x in contracts), frozenset([UrlContract, ReturnsContract])) # returns request for valid method diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4215ca56c..84f80d103 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -104,44 +104,44 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_dns_error(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @@ -187,25 +187,25 @@ foo body with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) - self.assertEqual(str(l).count("Got response 200"), 1) + self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) def _assert_retried(self, log): self.assertEqual(str(log).count("Retrying"), 2) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 745f1e5bf..c1e6f744b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -856,8 +856,7 @@ class S3TestCase(unittest.TestCase): def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. date = 'Tue, 27 Mar 2007 19:44:46 +0000' - req = Request('s3://johnsmith/?acl', - method='GET', headers={'Date': date}) + req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -882,8 +881,9 @@ class S3TestCase(unittest.TestCase): with self._mocked_date(date): httpreq = self.download_request(req, self.spider) # botocore does not override Date with x-amz-date - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + self.assertEqual( + httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index f86c50f50..d54434c8f 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check( + log.check( ('scrapy.downloadermiddlewares.cookies', 'DEBUG', 'Received cookies from: <200 http://scrapytest.org/>\n' @@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check() + log.check() def test_do_not_break_on_non_utf8_header(self): req = Request('http://scrapytest.org/') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 36743b1de..9841d7a76 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase): os.environ.pop('file_proxy', None) mw = HttpProxyMiddleware() - for url, proxy in [('http://e.com', http_proxy), - ('https://e.com', https_proxy), ('file://tmp/a', None)]: + for url, proxy in [ + ('http://e.com', http_proxy), + ('https://e.com', https_proxy), + ('file://tmp/a', None), + ]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEqual(req.url, url) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 61c9eddbc..22f23d7b5 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' req = Request(url, method='POST', body='test', - headers={'Content-Type': 'text/plain', 'Content-length': '4'}) + headers={'Content-Type': 'text/plain', 'Content-length': '4'}) rsp = Response(url, headers={'Location': url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e118750e3..29357ba94 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -81,9 +81,17 @@ class RetryTest(unittest.TestCase): assert self.crawler.stats.get_value('retry/count') == 2 def test_twistederrors(self): - exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, - DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost, ResponseFailed] + exceptions = [ + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + defer.TimeoutError, + DNSLookupError, + ResponseFailed, + TCPTimedOutError, + TimeoutError, + ] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 7426107c1..41a8d16bc 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase): shutil.rmtree(path) def test_log(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -177,15 +177,18 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' - no more duplicates will be shown' - ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' + ) + ) dupefilter.close('finished') def test_log_debug(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: None)'))) - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: http://scrapytest.org/INDEX.html)'))) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: None)' + ) + ) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: http://scrapytest.org/INDEX.html)' + ) + ) dupefilter.close('finished') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 77da15ce6..a672963f3 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -415,8 +415,7 @@ class FormRequestTest(RequestTest): # using multiples values for a single key data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual(r3.body, - b'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') def test_from_response_post(self): response = _buildresponse( @@ -426,8 +425,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -446,8 +444,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -468,8 +465,7 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", encoding='latin1', ) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -488,8 +484,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -549,8 +544,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - r1 = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(r1.method, 'GET') self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2f73afe56..f9b6b92f4 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -486,8 +486,10 @@ class TextResponseTest(BaseResponseTest): response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( - response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).getall(), + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1, + ).getall(), response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) @@ -567,12 +569,14 @@ class TextResponseTest(BaseResponseTest): 'http://example.com', body=b'''click me''' ) - self._assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self._assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) + self._assert_followed_url( + resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url( + resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) def test_follow_encoding(self): resp1 = self.response_class( diff --git a/tests/test_item.py b/tests/test_item.py index c4dcdbd42..60468971c 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase): item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') self.assertEqual(item['load'], 'Y') - self.assertEqual(D.fields, {'load': {'default': 'A'}, - 'save': {'default': 'A'}}) + self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}}) # D class inverted class E(C, B): @@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'load': {'default': 'C'}, - 'save': {'default': 'C'}}) + self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}}) def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(D(save='X')['save'], 'X') self.assertEqual(D(load='X')['load'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'C'}, - 'load': {'default': 'D'}, 'update': {'default': 'D'}}) + self.assertEqual( + D.fields, + {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}}) # D class inverted class E(C, B): @@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'C'}, - 'load': {'default': 'E'}, 'update': {'default': 'C'}}) + self.assertEqual( + E.fields, + {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}}) def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) # D class inverted class E(C, B): @@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) def test_to_dict(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 46d8c13af..2a2650480 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -172,9 +172,9 @@ class Base: self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), - allow_domains=('blah1.com',), - deny_domains=('blah2.com',)) + lx = self.extractor_cls(allow=['blah1'], deny=['blah2'], + allow_domains=['blah1.com'], + deny_domains=['blah2.com']) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) self.assertEqual(lx.matches('http://blah1.com/blah2'), False) self.assertEqual(lx.matches('http://blah2.com/blah1'), False) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index cd6cb8016..7064337ad 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) (referer: None)") + self.assertEqual(logline, "Crawled (200) (referer: None)") def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) ['test', 'flag'] (referer: None)") + self.assertEqual( + logline, + "Crawled (200) ['test', 'flag'] (referer: None)") def test_dropped(self): item = {} @@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: None) []") def test_crawled_without_referer(self): @@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Crawled (200) (referer: None) ['test', 'flag']") + self.assertEqual( + logline, + "Crawled (200) (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): diff --git a/tests/test_mail.py b/tests/test_mail.py index f5cb81a8b..53dbc0686 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - attachs=attachs, _callback=self._catch_mail_sent) + attachs=attachs, _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3af514bb0..3364d2258 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['open_spider']], [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['close_spider']], [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['process']], [M1, M3]) def test_enabled(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e2578a9c9..6d15aaf31 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -131,7 +131,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -140,7 +141,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -149,7 +151,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) @@ -163,7 +166,8 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 949f0dea1..19ff00350 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 1 - record = l.records[0] + assert len(log.records) == 1 + record = log.records[0] assert record.levelname == 'ERROR' self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 0 + assert len(log.records) == 0 @inlineCallbacks def test_default_process_item(self): @@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(True, rsp)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_downloaded', 'request_callback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed']) @inlineCallbacks def test_result_failure(self): @@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(False, fail)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_failed', 'request_errback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed']) @inlineCallbacks def test_mix_of_success_and_failure(self): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 188ec68dd..4763a5417 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index fda44653a..2da6aa4b5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase): 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) - self.assertDictEqual(s.copy_to_dict(), - {'HASNOBASE': {3: 3000}, - 'TEST': {1: 10, 3: 30}, - 'TEST_BASE': {1: 1, 2: 2}, - 'TEST_BOOLEAN': False, - 'TEST_LIST': [1, 2], - 'TEST_STRING': 'a string'}) + self.assertDictEqual( + s.copy_to_dict(), + { + 'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_STRING': 'a string', + } + ) def test_freeze(self): self.settings.freeze() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b6fb27ffe..265970b43 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual(set(self.spider_loader.list()), + self.assertEqual( + set(self.spider_loader.list()), set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): @@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase): self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), ['spider2']) - self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), + self.assertEqual( + set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), set(['spider1', 'spider2'])) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), ['spider3']) def test_load_spider_module(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 29584f21b..e032b247c 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual([], - self.mw.process_spider_exception(self.res404, - HttpError(self.res404), self.spider)) - self.assertEqual(None, - self.mw.process_spider_exception(self.res404, - Exception(), self.spider)) + self.assertEqual( + [], + self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) + self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEqual(None, - self.mw.process_spider_input(res, self.spider)) + self.assertIsNone(self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) class TestHttpErrorMiddlewareSettings(TestCase): @@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) - self.assertEqual(None, - self.mw.process_spider_input(self.res402, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) class TestHttpErrorMiddlewareIntegrational(TrialTestCase): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b96807bc2..0f4b98a07 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase): def test_process_spider_output(self): res = Response('http://scrapytest.org') - onsite_reqs = [Request('http://scrapytest.org/1'), - Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/'), - Request('http://scrapy.test.org:8000/')] - offsite_reqs = [Request('http://scrapy2.org'), - Request('http://offsite.tld/'), - Request('http://offsite.tld/scrapytest.org'), - Request('http://offsite.tld/rogue.scrapytest.org'), - Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org'), - Request('http://test.org/'), - Request('http://notscrapy.test.org/')] + onsite_reqs = [ + Request('http://scrapytest.org/1'), + Request('http://scrapy.org/1'), + Request('http://sub.scrapy.org/1'), + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/'), + Request('http://scrapy.test.org:8000/'), + ] + offsite_reqs = [ + Request('http://scrapy2.org'), + Request('http://offsite.tld/'), + Request('http://offsite.tld/scrapytest.org'), + Request('http://offsite.tld/rogue.scrapytest.org'), + Request('http://rogue.scrapytest.org.haha.com'), + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/'), + ] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2cf9135f..becacce62 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() @@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 332120021..e5d3ef582 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase): class UtilsConfTestCase(unittest.TestCase): def test_arglist_to_dict(self): - self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']), + self.assertEqual( + arglist_to_dict(['arg1=val1', 'arg2=val2']), {'arg1': 'val1', 'arg2': 'val2'}) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 69339256e..824170d71 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -46,8 +46,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').getall() - for e in self.xmliter(response, 'matchme...')] + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) def test_xmliter_unicode(self): @@ -358,15 +357,23 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'latin1', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + ] + ) response = TextResponse(url="http://example.com/", body=body2, encoding='cp852') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'cp852', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + ] + ) class TestHelper(unittest.TestCase): diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 25cd904bc..535f56691 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase): def test_top_level_logger(self): logger = logging.getLogger('test') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_children_logger(self): logger = logging.getLogger('test.test1') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_overlapping_name_logger(self): logger = logging.getLogger('test2') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test2', 'WARNING', 'test log msg')) + log.check(('test2', 'WARNING', 'test log msg')) def test_different_name_logger(self): logger = logging.getLogger('different') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('different', 'WARNING', 'test log msg')) + log.check(('different', 'WARNING', 'test log msg')) class LogCounterHandlerTest(unittest.TestCase): @@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase): sys.stdout = self.stdout def test_redirect(self): - with LogCapture() as l: + with LogCapture() as log: print('test log msg') - l.check(('test', 'ERROR', 'test log msg')) + log.check(('test', 'ERROR', 'test log msg')) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 015a0e5a2..28205e0d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c']) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 50b026d1c..450e4bdca 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def test_all_attributes(self): - r = Request("http://www.example.com", + r = Request( + url="http://www.example.com", callback=self.spider.parse_item, errback=self.spider.handle_error, method="POST", diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 45f0f59e4..50efb63ca 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase): self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, include_headers=['Accept-Language'])) - self.assertNotEqual(request_fingerprint(r1), - request_fingerprint(r2, include_headers=['Accept-Language'])) + self.assertNotEqual( + request_fingerprint(r1), + request_fingerprint(r2, include_headers=['Accept-Language'])) self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']), request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language'])) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index c83c9398c..b66588efb 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase): dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as l: + with LogCapture() as log: result = yield defer.maybeDeferred( self._get_result, test_signal, arg='test', handlers_called=handlers_called @@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) @@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase): test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as l: + with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(l.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(l)) + self.assertEqual(len(log.records), 1) + self.assertIn("Cannot return deferreds from signal handler", str(log)) dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 08b215434..d77978ff1 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase): """) assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + ] + ) def test_sitemap_index(self): s = Sitemap(b""" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 307fadb5c..188e54602 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): return f from twisted.web.client import _makeGetterFactory - return _makeGetterFactory(to_bytes(url), _clientfactory, - contextFactory=contextFactory, *args, **kwargs).deferred + return _makeGetterFactory( + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + ).deferred class ParseUrlTestCase(unittest.TestCase): @@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'Content-Length': '12981', 'Useful': 'value'})) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Content-Length: 9\r\n" b"Useful: value\r\n" @@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") @@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): body='name=value', headers={'Content-Type': 'application/x-www-form-urlencoded'})) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Connection: close\r\n" @@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): url='http://foo/bar' )) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" @@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }, )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }), )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual(protocol.headers, - Headers({'Hello': ['World'], 'Foo': ['Bar']})) + self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) class EncodingResource(resource.Resource): @@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEqual(pageData, - b'\n\n \n \n' - b' \n \n ' - b'click here\n \n\n') + self.assertEqual( + pageData, + b'\n\n \n \n' + b' \n \n ' + b'click here\n \n\n') def test_encoding(self): """ Test that non-standart body encoding matches @@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) - return getPage(self.getURL("payload"), body=s, - contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s)) + return getPage( + self.getURL("payload"), body=s, contextFactory=client_context_factory + ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDefaultCiphers(self): s = "0123456789" * 10