From f9a29f03d9a0eb9173a91f225177b7bee7d382c9 Mon Sep 17 00:00:00 2001 From: Mark Mayo Date: Sun, 27 Nov 2022 23:00:13 +1300 Subject: [PATCH] Address some issues reported by Pylint (#5677) Co-authored-by: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Co-authored-by: Andrey Rahmatullin --- conftest.py | 2 +- docs/_ext/scrapydocs.py | 2 +- docs/conftest.py | 8 +++-- docs/utils/linkfix.py | 9 +++--- pylintrc | 27 +---------------- scrapy/commands/__init__.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/commands/parse.py | 2 -- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 22 +++++++------- scrapy/core/downloader/tls.py | 1 - scrapy/core/downloader/webclient.py | 4 +-- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/scheduler.py | 4 +-- scrapy/core/scraper.py | 30 +++++++++---------- scrapy/core/spidermw.py | 21 ++++++------- scrapy/downloadermiddlewares/cookies.py | 3 +- .../downloadermiddlewares/httpcompression.py | 1 - scrapy/downloadermiddlewares/redirect.py | 8 ++--- scrapy/downloadermiddlewares/retry.py | 17 +++++------ scrapy/downloadermiddlewares/robotstxt.py | 3 +- scrapy/downloadermiddlewares/stats.py | 4 +-- scrapy/dupefilters.py | 2 +- scrapy/exporters.py | 4 +-- scrapy/extensions/httpcache.py | 14 ++++----- scrapy/extensions/telnet.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/headers.py | 7 ++--- scrapy/http/request/__init__.py | 2 +- scrapy/http/request/form.py | 4 +-- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/__init__.py | 2 +- scrapy/pipelines/files.py | 6 ++-- scrapy/responsetypes.py | 10 +++---- scrapy/robotstxt.py | 1 - scrapy/settings/__init__.py | 6 ++-- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 5 ++-- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/sitemap.py | 4 +-- scrapy/utils/conf.py | 14 ++++----- scrapy/utils/defer.py | 22 ++++++-------- scrapy/utils/display.py | 2 +- scrapy/utils/ftp.py | 1 - scrapy/utils/iterators.py | 13 ++++---- scrapy/utils/misc.py | 15 ++++------ scrapy/utils/python.py | 5 ++-- scrapy/utils/response.py | 6 ++-- scrapy/utils/serialize.py | 19 ++++++------ scrapy/utils/spider.py | 5 ++-- setup.py | 2 +- tests/CrawlerProcess/reactor_default.py | 3 +- .../reactor_default_twisted_reactor_select.py | 3 +- tests/CrawlerProcess/reactor_select.py | 4 ++- ..._select_subclass_twisted_reactor_select.py | 4 +-- .../reactor_select_twisted_reactor_select.py | 4 ++- tests/CrawlerRunner/ip_address.py | 8 ++--- tests/mockserver.py | 3 +- tests/test_command_check.py | 4 +-- tests/test_command_parse.py | 8 ++--- tests/test_commands.py | 28 ++++++++--------- tests/test_crawler.py | 5 ++-- tests/test_downloader_handlers.py | 5 ++-- tests/test_downloader_handlers_http2.py | 6 ++-- tests/test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 7 ++--- ...st_downloadermiddleware_httpcompression.py | 3 +- tests/test_exporters.py | 6 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_http_request.py | 2 +- tests/test_http_response.py | 2 +- tests/test_loader_deprecated.py | 5 ++-- tests/test_pipeline_media.py | 4 +-- tests/test_request_attribute_binding.py | 4 +-- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler_base.py | 2 -- tests/test_spider.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 1 - tests/test_spidermiddleware_referer.py | 4 +-- tests/test_squeues_request.py | 1 - tests/test_utils_defer.py | 5 ++-- tests/test_utils_deprecate.py | 1 + tests/test_webclient.py | 2 +- 87 files changed, 219 insertions(+), 289 deletions(-) mode change 100755 => 100644 docs/utils/linkfix.py diff --git a/conftest.py b/conftest.py index 7c1da3556..2a5d55083 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path('tests/ignores.txt').open() as reader: +with Path('tests/ignores.txt').open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index f0f382da3..337604cf1 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,8 +1,8 @@ +from operator import itemgetter from docutils.parsers.rst.roles import set_classes from docutils import nodes from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode -from operator import itemgetter class settingslist_node(nodes.General, nodes.Element): diff --git a/docs/conftest.py b/docs/conftest.py index 24a72a4b6..a6dacd265 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,15 +1,17 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from pathlib import Path -from scrapy.http.response.html import HtmlResponse from sybil import Sybil +from sybil.parsers.doctest import DocTestParser +from sybil.parsers.skip import skip + try: # >2.0.1 from sybil.parsers.codeblock import PythonCodeBlockParser except ImportError: from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser -from sybil.parsers.doctest import DocTestParser -from sybil.parsers.skip import skip + +from scrapy.http.response.html import HtmlResponse def load_response(url: str, filename: str) -> HtmlResponse: diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py old mode 100755 new mode 100644 index 7a0c5288c..a0d9a1cbd --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -13,6 +13,7 @@ Author: dufferzafar """ import re +import sys from pathlib import Path @@ -28,11 +29,11 @@ def main(): # Read lines from the linkcheck output file try: - with Path("build/linkcheck/output.txt").open() as out: + with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() except IOError: print("linkcheck output not found; please run linkcheck first.") - exit(1) + sys.exit(1) # For every line, fix the respective file for line in output_lines: @@ -52,12 +53,12 @@ def main(): # Update the previous file if _filename: - Path(_filename).write_text(_contents) + Path(_filename).write_text(_contents, encoding="utf-8") _filename = newfilename # Read the new file to memory - _contents = Path(_filename).read_text() + _contents = Path(_filename).read_text(encoding="utf-8") _contents = _contents.replace(match.group(3), match.group(4)) else: diff --git a/pylintrc b/pylintrc index 18819feba..0a2276fb8 100644 --- a/pylintrc +++ b/pylintrc @@ -9,27 +9,19 @@ disable=abstract-method, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-indentation, bad-mcs-classmethod-argument, - bad-super-call, bare-except, - blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, - consider-iterating-dictionary, consider-using-dict-items, - consider-using-from-import, consider-using-in, - consider-using-set-comprehension, - consider-using-sys-exit, consider-using-with, cyclic-import, dangerous-default-value, - deprecated-method, - deprecated-module, + disallowed-name, duplicate-code, # https://github.com/PyCQA/pylint/issues/214 eval-used, expression-not-assigned, @@ -52,18 +44,12 @@ disable=abstract-method, lost-exception, method-hidden, missing-docstring, - missing-final-newline, - multiple-imports, - multiple-statements, - no-else-continue, no-else-raise, - no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, - not-an-iterable, not-callable, pointless-statement, pointless-string-statement, @@ -74,10 +60,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, - singleton-comparison, super-init-not-called, - super-with-arguments, - superfluous-parens, too-few-public-methods, too-many-ancestors, too-many-arguments, @@ -89,31 +72,23 @@ disable=abstract-method, too-many-locals, too-many-public-methods, too-many-return-statements, - trailing-newlines, - trailing-whitespace, unbalanced-tuple-unpacking, undefined-variable, undefined-loop-variable, unexpected-special-method-signature, - ungrouped-imports, - unidiomatic-typecheck, unnecessary-comprehension, unnecessary-dunder-call, - unnecessary-lambda, unnecessary-pass, unreachable, - unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, unused-private-member, unused-variable, unused-wildcard-import, - use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, useless-super-delegation, wildcard-import, - wrong-import-order, wrong-import-position diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index c4d4bebb2..d0fb4efd8 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -95,7 +95,7 @@ class ScrapyCommand: self.settings.set('LOG_ENABLED', False, priority='cmdline') if opts.pidfile: - Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep) + Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8") if opts.pdb: failure.startDebugMode() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index ed9660252..b7c2f85fb 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -63,7 +63,7 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - print(template_file.read_text()) + print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: raise UsageError() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index d93ab2ac5..c2df7f8a5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,14 +6,12 @@ from itemadapter import is_item, ItemAdapter from w3lib.url import is_url from twisted.internet.defer import maybeDeferred - from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index c0146a0a6..a75532d2a 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -33,5 +33,4 @@ class HTTP10DownloadHandler: crawler=self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) - else: - return reactor.connectTCP(host, port, factory) + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f07f0780e..1c98e60e1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -26,7 +26,6 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -289,16 +288,15 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - else: - proxyScheme = proxyScheme or b'http' - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) - return self._ProxyAgent( - reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding='ascii'), - connectTimeout=timeout, - bindAddress=bindaddress, - pool=self._pool, - ) + proxyScheme = proxyScheme or b'http' + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) + return self._ProxyAgent( + reactor=reactor, + proxyURI=to_bytes(proxyURI, encoding='ascii'), + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) return self._Agent( reactor=reactor, @@ -567,7 +565,7 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["dataloss"]) return - elif not self._fail_on_dataloss_warned: + if not self._fail_on_dataloss_warned: logger.warning("Got data loss in %s. If you want to process broken " "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" " -- This message won't be shown in further requests", diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 698a1c85c..7d67a426f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -7,7 +7,6 @@ from twisted.internet.ssl import AcceptableCiphers from scrapy.utils.ssl import x509name_to_string, get_temp_key_info - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 7d048c1e4..255ca62e6 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,8 +1,8 @@ import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag - from twisted.web.http import HTTPClient + from twisted.internet import defer from twisted.internet.protocol import ClientFactory @@ -185,7 +185,7 @@ class ScrapyHTTPClientFactory(ClientFactory): @param version: The HTTP version. @type version: L{bytes} @param status: The HTTP status code, an integer represented as a - bytestring. + bytestring. @type status: L{bytes} @param message: The HTTP status message. @type message: L{bytes} diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f7b0c3f99..c6b357be3 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -140,7 +140,7 @@ class ScrapyProxyH2Agent(H2Agent): connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None, ) -> None: - super(ScrapyProxyH2Agent, self).__init__( + super().__init__( reactor=reactor, pool=pool, context_factory=context_factory, diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1d150b7ce..6047f9ca8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -42,7 +42,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") + return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}" class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 366449f51..6dd5d51a8 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -333,9 +333,9 @@ class Scheduler(BaseScheduler): path = Path(dqdir, 'active.json') if not path.exists(): return [] - with path.open() as f: + with path.open(encoding="utf-8") as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with Path(dqdir, 'active.json').open('w') as f: + with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e1fdd8d13..7225e0743 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -152,9 +152,9 @@ class Scraper: """ if isinstance(result, Response): return self.spidermw.scrape_response(self.call_spider, result, request, spider) - else: # result is a Failure - dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + # else result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): @@ -276,17 +276,15 @@ class Scraper: return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) - else: - logkws = self.logformatter.item_error(item, ex, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, - exc_info=failure_to_exc_info(output)) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, item=item, response=response, - spider=spider, failure=output) - else: - logkws = self.logformatter.scraped(output, response, spider) - if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logkws = self.logformatter.item_error(item, ex, response, spider) + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, + exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, - spider=spider) + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) + logkws = self.logformatter.scraped(output, response, spider) + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_scraped, item=output, response=response, + spider=spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aa02f29f..91e4b9cb4 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -121,11 +121,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading return dfd.result - else: - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" - raise _InvalidOutput(msg) + # we forbid waiting here because otherwise we would need to return a deferred from + # _process_spider_exception too, which complicates the architecture + msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + raise _InvalidOutput(msg) elif result is None: continue else: @@ -213,8 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if last_result_is_async: return MutableAsyncChain(result, recovered) - else: - return MutableChain(result, recovered) # type: ignore[arg-type] + return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] ) -> Union[MutableChain, MutableAsyncChain]: @@ -227,11 +225,10 @@ class SpiderMiddlewareManager(MiddlewareManager): result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) - else: - if isinstance(recovered, AsyncIterable): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) # type: ignore[arg-type] + if isinstance(recovered, AsyncIterable): + recovered_collected = await collect_asyncgen(recovered) + recovered = MutableChain(recovered_collected) + return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index c592acb57..3cba269f1 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - elif isinstance(request.cookies, dict): + if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 4e7feeeaf..e57f6fcf8 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -8,7 +8,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip - ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] try: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index c8c84ffb2..fba825947 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -8,7 +8,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured - logger = logging.getLogger(__name__) @@ -56,10 +55,9 @@ class BaseRedirectMiddleware: {'reason': reason, 'redirected': redirected, 'request': request}, extra={'spider': spider}) return redirected - else: - logger.debug("Discarding %(request)s: max redirections reached", - {'request': request}, extra={'spider': spider}) - raise IgnoreRequest("max redirections reached") + logger.debug("Discarding %(request)s: max redirections reached", + {'request': request}, extra={'spider': spider}) + raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): redirect_request = _build_redirect_request( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c6cc7c56d..2de610e86 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -113,15 +113,14 @@ def get_retry_request( stats.inc_value(f'{stats_base_key}/count') stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request - else: - stats.inc_value(f'{stats_base_key}/max_reached') - logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " - "%(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}, - ) - return None + stats.inc_value(f'{stats_base_key}/max_reached') + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) + return None class RetryMiddleware: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index e66bf177e..7bd39aa43 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -81,8 +81,7 @@ class RobotsTxtMiddleware: return result self._parsers[netloc].addCallback(cb) return d - else: - return self._parsers[netloc] + return self._parsers[netloc] def _logerror(self, failure, request, spider): if failure.type is not IgnoreRequest: diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 25fb1ed9d..2bb022c16 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,9 +1,9 @@ +from twisted.web import http + from scrapy.exceptions import NotConfigured from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from twisted.web import http - def get_header_size(headers): size = 0 diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 2b8b09614..6dca11d31 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, 'requests.seen').open('a+') + self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8") self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..2135c3bbc 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -334,9 +334,9 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value): if isinstance(value, Item): return self.export_item(value) - elif is_item(value): + if is_item(value): return dict(self._serialize_item(value)) - elif is_listlike(value): + if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 58c4e644e..4e76fe5e5 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -17,7 +17,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -81,25 +80,24 @@ class RFC2616Policy: if b'no-store' in cc: return False # Never cache 304 (Not Modified) responses - elif response.status == 304: + if response.status == 304: return False # Cache unconditionally if configured to do so - elif self.always_store: + if self.always_store: return True # Any hint on response expiration is good - elif b'max-age' in cc or b'Expires' in response.headers: + if b'max-age' in cc or b'Expires' in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - elif response.status in (300, 301, 308): + if response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator - elif response.status in (200, 203, 401): + if response.status in (200, 203, 401): return b'Last-Modified' in response.headers or b'ETag' in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated - else: - return False + return False def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 1663604e7..054350e4a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -11,6 +11,7 @@ import binascii import os from twisted.internet import protocol + try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults @@ -26,7 +27,6 @@ from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.decorators import defers - logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 56e5ad2d2..eb21e426e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -56,7 +56,7 @@ class AutoThrottle: { 'slot': key, 'concurrency': conc, 'delay': slot.delay * 1000, 'delaydiff': diff * 1000, - 'latency': latency * 1000, 'size': size + 'latency': latency * 1000, 'size': size, }, extra={'spider': spider} ) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 9c03fe54f..2e0020890 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -37,12 +37,11 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, str): + if isinstance(x, str): return x.encode(self.encoding) - elif isinstance(x, int): + if isinstance(x, int): return str(x).encode(self.encoding) - else: - raise TypeError(f'Unsupported value type: {type(x)}') + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1ececaf1d..a1001fc4a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -185,7 +185,7 @@ class Request(object_ref): } for attr in self.attributes: d.setdefault(attr, getattr(self, attr)) - if type(self) is not Request: + if type(self) is not Request: # pylint: disable=unidiomatic-typecheck d["_class"] = self.__module__ + '.' + self.__class__.__name__ return d diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0c947565a..8e0a7fae2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -190,7 +190,7 @@ def _select_value(ele: SelectElement, n: str, v: str): # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) - elif v is not None and multiple: + if v is not None and multiple: # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') @@ -236,7 +236,7 @@ def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tup el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') - elif len(el) > 1: + if len(el) > 1: raise ValueError(f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}") else: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 55639f504..1e6ab984a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -import lxml.etree as etree +from lxml import etree from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index b5725a8ee..c42dd423e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -18,7 +18,7 @@ class ItemPipelineManager(MiddlewareManager): return build_component_list(settings.getwithbase('ITEM_PIPELINES')) def _add_middleware(self, pipe): - super(ItemPipelineManager, self)._add_middleware(pipe) + super()._add_middleware(pipe) if hasattr(pipe, 'process_item'): self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4e0211d1b..51aedafe8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -221,16 +221,14 @@ class GCSFilesStore: checksum = blob.md5_hash last_modified = time.mktime(blob.updated.timetuple()) return {'checksum': checksum, 'last_modified': last_modified} - else: - return {} + return {} blob_path = self._get_blob_path(path) return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: return headers['Content-Type'] - else: - return 'application/octet-stream' + return 'application/octet-stream' def _get_blob_path(self, path): return self.prefix + path diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3efd4d2fd..a34d7d25f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -42,11 +42,10 @@ class ResponseTypes: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response - elif mimetype in self.classes: + if mimetype in self.classes: return self.classes[mimetype] - else: - basetype = f"{mimetype.split('/')[0]}/*" - return self.classes.get(basetype, Response) + basetype = f"{mimetype.split('/')[0]}/*" + return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type @@ -83,8 +82,7 @@ class ResponseTypes: mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) - else: - return Response + return Response def from_body(self, body): """Try to guess the appropriate response based on the body content. diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f8649e56b..1d40f0484 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 43ee433d1..b540e6182 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -24,8 +24,7 @@ def get_settings_priority(priority): """ if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] - else: - return priority + return priority class SettingsAttribute: @@ -260,8 +259,7 @@ class BaseSettings(MutableMapping): """ if len(self) > 0: return max(self.getpriority(name) for name in self) - else: - return get_settings_priority('default') + return get_settings_priority('default') def __setitem__(self, name, value): self.set(name, value) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 448bc1367..98a186510 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -65,7 +65,7 @@ class OffsiteMiddleware: for domain in allowed_domains: if domain is None: continue - elif url_pattern.match(domain): + if url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8027beb92..4a6c4de5e 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -189,8 +189,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - else: - return origin + return origin class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -216,7 +215,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ( + if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2d9328633..b514ed9ce 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -26,7 +26,7 @@ def _identity_process_request(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, str): + if isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1f72e76b7..43d138753 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -69,7 +69,7 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif gzip_magic_number(response): + if gzip_magic_number(response): return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) @@ -80,7 +80,7 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): + if response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 126ead020..8ea42ce75 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -35,9 +35,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): else: compbs.set(convert(k), v, priority=prio) return compbs - else: - _check_components(compdict) - return {convert(k): v for k, v in compdict.items()} + _check_components(compdict) + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" @@ -181,11 +180,10 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} - else: - raise UsageError( - 'The -t command-line option cannot be used if multiple output ' - 'URIs are specified' - ) + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result: Dict[str, Dict[str, Any]] = {} for element in output: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 38aefd6d0..ddacfaa49 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -58,10 +58,9 @@ def defer_succeed(result) -> Deferred: def defer_result(result) -> Deferred: if isinstance(result, Deferred): return result - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer_fail(result) - else: - return defer_succeed(result) + return defer_succeed(result) def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: @@ -267,10 +266,9 @@ def deferred_from_coro(o) -> Any: # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return ensureDeferred(o) - else: - # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = get_asyncio_event_loop_policy().get_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -295,12 +293,11 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: if isinstance(result, Deferred): return result - elif asyncio.isfuture(result) or inspect.isawaitable(result): + if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer.fail(result) - else: - return defer.succeed(result) + return defer.succeed(result) def deferred_to_future(d: Deferred) -> Future: @@ -352,5 +349,4 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ if not is_asyncio_reactor_installed(): return d - else: - return deferred_to_future(d) + return deferred_to_future(d) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index d28df40c7..64969cb13 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,8 +5,8 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from packaging.version import Version as parse_version from pprint import pformat as pformat_ +from packaging.version import Version as parse_version def _enable_windows_terminal_processing(): diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6cace4f07..775b25ea8 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,5 +1,4 @@ import posixpath - from ftplib import error_perm, FTP from posixpath import dirname diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 3b504e56a..da77ca46f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -138,8 +138,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): {'csvlnum': csv_r.line_num, 'csvrow': len(row), 'csvheader': len(headers)}) continue - else: - yield dict(zip(headers, row)) + yield dict(zip(headers, row)) def _body_or_str(obj, unicode=True): @@ -152,11 +151,9 @@ def _body_or_str(obj, unicode=True): if isinstance(obj, Response): if not unicode: return obj.body - elif isinstance(obj, TextResponse): + if isinstance(obj, TextResponse): return obj.text - else: - return obj.body.decode('utf-8') - elif isinstance(obj, str): + return obj.body.decode('utf-8') + if isinstance(obj, str): return obj if unicode else obj.encode('utf-8') - else: - return obj.decode('utf-8') if unicode else obj + return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e0f7ca9e5..1e0342ace 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -30,10 +30,9 @@ def arg_to_iter(arg): """ if arg is None: return [] - elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): + if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): return arg - else: - return [arg] + return [arg] def load_object(path): @@ -49,9 +48,8 @@ def load_object(path): if not isinstance(path, str): if callable(path): return path - else: - raise TypeError("Unexpected argument type, expected string " - f"or object, got: {type(path)}") + raise TypeError("Unexpected argument type, expected string " + f"or object, got: {type(path)}") try: dot = path.rindex('.') @@ -115,9 +113,8 @@ def extract_regex(regex, text, encoding='utf-8'): if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] - else: - return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) - for s in strings] + return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) + for s in strings] def md5sum(file): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0d9fdbf23..9df1c91de 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -187,10 +187,9 @@ def get_func_args(func, stripself=False): elif hasattr(func, '__call__'): if inspect.isroutine(func): return [] - elif getattr(func, '__name__', None) == '__call__': + if getattr(func, '__name__', None) == '__call__': return [] - else: - return get_func_args(func.__call__, True) + return get_func_args(func.__call__, True) else: raise TypeError(f'{type(func)} is not callable') if stripself: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 741dce350..23bd2da65 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,14 +9,14 @@ import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary +from twisted.web import http +from w3lib import html import scrapy from scrapy.http.response import Response -from twisted.web import http + from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.decorators import deprecated -from w3lib import html - _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index a73cf03c5..3602043f3 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -16,24 +16,23 @@ class ScrapyJSONEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, set): return list(o) - elif isinstance(o, datetime.datetime): + if isinstance(o, datetime.datetime): return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") - elif isinstance(o, datetime.date): + if isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) - elif isinstance(o, datetime.time): + if isinstance(o, datetime.time): return o.strftime(self.TIME_FORMAT) - elif isinstance(o, decimal.Decimal): + if isinstance(o, decimal.Decimal): return str(o) - elif isinstance(o, defer.Deferred): + if isinstance(o, defer.Deferred): return str(o) - elif is_item(o): + if is_item(o): return ItemAdapter(o).asdict() - elif isinstance(o, Request): + if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" - elif isinstance(o, Response): + if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" - else: - return super().default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index d0fd1757d..b2da69404 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -12,12 +12,11 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): if inspect.isasyncgen(result): return result - elif inspect.iscoroutine(result): + if inspect.iscoroutine(result): d = deferred_from_coro(result) d.addCallback(iterate_spider_output) return d - else: - return arg_to_iter(deferred_from_coro(result)) + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/setup.py b/setup.py index e413ea6e4..82ac86cdd 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( 'Tracker': 'https://github.com/scrapy/scrapy/issues', }, description='A high-level Web Crawling and Web Scraping framework', - long_description=open('README.rst').read(), + long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 2c867df61..06b849de3 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c2b30b044..a6dff0e05 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index ca70c06a0..907f50784 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 0035daf1e..6f48da691 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,7 +1,7 @@ -import scrapy -from scrapy.crawler import CrawlerProcess from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor +import scrapy +from scrapy.crawler import CrawlerProcess class SelectReactorSubclass(SelectReactor): diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 4f8394edb..c422b13ff 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index f545de39f..417294447 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -8,7 +8,6 @@ from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging - from tests.mockserver import MockServer, MockDNSServer @@ -30,9 +29,10 @@ class LocalhostSpider(Spider): def parse(self, response): netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) + host = netloc.split(":")[0] + self.logger.info(f"Host: {host}") + self.logger.info(f"Type: {type(response.ip_address)}") + self.logger.info(f"IP address: {response.ip_address}") if __name__ == "__main__": diff --git a/tests/mockserver.py b/tests/mockserver.py index 7916798f7..6d2d95692 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -29,8 +29,7 @@ def getarg(request, name, default=None, type=None): if type is not None: value = type(value) return value - else: - return default + return default # most of the following resources are copied from twisted.web.test.test_webclient diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 4077a9bce..049076e5c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -6,7 +6,7 @@ class CheckCommandTest(CommandTest): command = 'check' def setUp(self): - super(CheckCommandTest, self).setUp() + super().setUp() self.spider_name = 'check_spider' self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() @@ -24,7 +24,7 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """) + """, encoding="utf-8") def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 4053c95d5..368b80513 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -92,7 +92,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""") +""", encoding="utf-8") (self.proj_mod_path / 'pipelines.py').write_text(""" import logging @@ -103,9 +103,9 @@ class MyPipeline: def process_item(self, item, spider): logging.info('It Works!') return item -""") +""", encoding="utf-8") - with (self.proj_mod_path / 'settings.py').open("a") as f: + with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f: f.write(f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """) @@ -256,7 +256,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertTrue(file_path.is_file()) content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(), content) + self.assertEqual(file_path.read_text(encoding="utf-8"), content) def test_parse_add_options(self): command = parse.Command() diff --git a/tests/test_commands.py b/tests/test_commands.py index ea4cde363..91476abf8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -107,7 +107,7 @@ class ProjectTest(unittest.TestCase): def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with Path(filename).open("r") as f: + with Path(filename).open("r", encoding="utf-8") as f: for line in f: match = pattern.search(line) if match is not None: @@ -475,7 +475,7 @@ class GenspiderCommandTest(CommandTest): assert file_path.exists() # change name of spider but not its file name - with file_path.open('r+') as spider_file: + with file_path.open('r+', encoding="utf-8") as spider_file: file_data = spider_file.read() file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") spider_file.seek(0) @@ -489,14 +489,14 @@ class GenspiderCommandTest(CommandTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_filename_as_existing_spider_force(self): @@ -536,7 +536,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert file_path.exists() modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text() + file_contents_before = file_path.read_text(encoding="utf-8") if force: # use different template to ensure contents were changed @@ -544,14 +544,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_name_as_existing_file_force(self): @@ -596,7 +596,7 @@ class BadSpider(scrapy.Spider): fname = (tmpdir / name).resolve() else: fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content) + fname.write_text(content, encoding="utf-8") try: yield str(fname) finally: @@ -754,11 +754,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -798,7 +798,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' def setUp(self): - super(WindowsRunSpiderCommandTest, self).setUp() + super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name='badspider.pyw') @@ -860,7 +860,7 @@ class ViewCommandTest(CommandTest): class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): - Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code) + Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8") return self.proc('crawl', 'myspider', *args) def get_log(self, code, args=()): @@ -912,11 +912,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c09f1a6f2..e0902fdbe 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -11,6 +11,9 @@ from twisted.internet import defer from twisted.python.versions import Version from twisted.trial import unittest +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,8 +26,6 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0334be743..3dc2745a0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -110,7 +110,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly self.tmpname = Path(self.mktemp() + '^') - Path(self.tmpname).write_text('0123456789') + Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request @@ -722,8 +722,7 @@ class UriResource(resource.Resource): # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b'CONNECT': return request.uri - else: - return b'' + return b'' class HttpProxyTestCase(unittest.TestCase): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 079267535..c2fa3ec57 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -171,7 +171,7 @@ class Https2InvalidDNSId(Https2TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https2InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -190,7 +190,7 @@ class Https2InvalidDNSPattern(Https2TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https2InvalidDNSPattern, self).setUp() + super().setUp() @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") @@ -245,4 +245,4 @@ class Https2ProxyTestCase(Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): - yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() + yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index ba7453255..dd835b9c9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,6 +1,6 @@ import logging -from testfixtures import LogCapture from unittest import TestCase +from testfixtures import LogCapture import pytest diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index b2b5ce77d..1c4cae6d1 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -2,8 +2,8 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware from scrapy.spiders import Spider -from tests import get_testdata from scrapy.utils.test import assert_samelines +from tests import get_testdata def _test_data(formats): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 928c007f5..b3d8264ba 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -272,10 +272,9 @@ class RFC2616PolicyTest(DefaultStorageTest): if result: assert isinstance(result, (Request, Response)) return result - else: - result = mw.process_response(request, response, self.spider) - assert isinstance(result, Response) - return result + result = mw.process_response(request, response, self.spider) + assert isinstance(result, Response) + return result except Exception: print('Request', request) print('Response', response) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 6f4e217e6..b0272143d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest import TestCase, SkipTest from warnings import catch_warnings +from w3lib.encoding import resolve_encoding from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS @@ -12,8 +13,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -from w3lib.encoding import resolve_encoding - SAMPLEDIR = Path(tests_datadir, 'compressed') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 69ac928c3..86c5d8b5e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -400,8 +400,7 @@ class XmlItemExporterTest(BaseItemExporterTest): children = list(elem.iterchildren()) if children: return [(child.tag, sorted(xmltuple(child))) for child in children] - else: - return [(elem.tag, [(elem.text, ())])] + return [(elem.tag, [(elem.text, ())])] def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) @@ -621,8 +620,7 @@ class CustomExporterItemTest(unittest.TestCase): def serialize_field(self, field, name, value): if name == 'age': return str(int(value) + 1) - else: - return super().serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 402348cf9..e461bcf2c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -164,7 +164,7 @@ class RequestHeaders(LeafResource): def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: - pem = key_file.read_text() + certificate_file.read_text() + pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 579ef9fa2..9f7f1854f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -223,7 +223,7 @@ class RequestTest(unittest.TestCase): r1 = CustomRequest('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomRequest + assert isinstance(r2, CustomRequest) def test_replace(self): """Test Request.replace() method""" diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b42c95045..f51f3d988 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase): r1 = CustomResponse('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomResponse + assert isinstance(r2, CustomResponse) def test_replace(self): """Test Response.replace() method""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 0fd52da5f..14b3b5568 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -391,10 +391,9 @@ class BasicItemLoaderTest(unittest.TestCase): def join(values, sep=None, loader_context=None, ignored=None): if sep is not None: return sep.join(values) - elif loader_context and 'sep' in loader_context: + if loader_context and 'sep' in loader_context: return loader_context['sep'].join(values) - else: - return ''.join(values) + return ''.join(values) class TestItemLoader(NameItemLoader): name_out = Compose(partial(join, sep='+')) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 84e867660..0a94ae699 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -388,11 +388,11 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): self._mockcalled.append('thumb_path') - return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) + return super().thumb_path(request, thumb_id, response, info) def get_images(self, response, request, info): self._mockcalled.append('get_images') - return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) + return super().get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 0406d906f..15e400327 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,12 +1,12 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase +from testfixtures import LogCapture + from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 002a04358..63e3aac00 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -91,7 +91,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(kwargs['callback'] == 'some_callback') self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): - self.checks.append(kwargs == {}) + self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison self.crawler.stats.inc_value('boolean_checks') def parse_no_kwargs(self, response): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index fc234a83d..6602b626d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -11,10 +11,8 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer - PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] diff --git a/tests/test_spider.py b/tests/test_spider.py index e1527620f..cb66066b0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -7,6 +7,7 @@ from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest +from w3lib.url import safe_url_string from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse @@ -22,7 +23,6 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 697105967..0b6b51a5b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import shutil import warnings from pathlib import Path +import tempfile from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -10,7 +11,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy -import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 46f74ae52..ee11ee492 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -6,11 +6,11 @@ from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.mockserver import MockServer from tests.spiders import MockServerSpider diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dac246fb6..8dd1def17 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -4,7 +4,6 @@ from twisted.trial.unittest import TestCase from scrapy import Request, Spider from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9456b01d4..63daf0b8a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,8 +1,8 @@ from urllib.parse import urlparse from unittest import TestCase import warnings - from scrapy.http import Response, Request + from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware @@ -380,7 +380,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' - elif scheme == 'http': + if scheme == 'http': return b'http://python.org/' diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index c5fcc1853..22dafb2d2 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -16,7 +16,6 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler - """ Queues that handle requests """ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d39de7430..97c1c60d1 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -193,9 +193,8 @@ class AsyncCooperatorTest(unittest.TestCase): delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) return dfd - else: - # simulate trivial sync processing - results.append(o) + # simulate trivial sync processing + results.append(o) @staticmethod def get_async_iterable(length): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index e47afa266..50c63dfab 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -2,6 +2,7 @@ import inspect import unittest from unittest import mock import warnings + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 69d9a9e3a..3dcaf71dd 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -47,7 +47,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): from twisted.web.client import _makeGetterFactory return _makeGetterFactory( - to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs, ).deferred