From 50500a6b2897418405b459b6a2a80a7a5b0a6f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 17:14:18 +0100 Subject: [PATCH 01/33] Implement a NO_CALLBACK value for Request.callback --- docs/topics/request-response.rst | 43 ++++++++++++++------ scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/http/request/__init__.py | 28 ++++++++++--- scrapy/pipelines/media.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 8 ++++ tests/test_http_request.py | 9 ++++ tests/test_pipeline_files.py | 5 +-- tests/test_pipeline_images.py | 7 +--- tests/test_pipeline_media.py | 2 + 9 files changed, 79 insertions(+), 30 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a0d9fc03e..766710d66 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -32,11 +32,22 @@ Request objects :type url: str :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. For more information - see :ref:`topics-request-response-ref-request-callback-arguments` below. - If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.Spider.parse` method will be used. - Note that if exceptions are raised during processing, errback is called instead. + request (once it's downloaded) as its first parameter. + + In addition to a function, the following values are supported: + + - ``None`` (default), which indicates that the spider's + :meth:`~scrapy.Spider.parse` method must be used. + + - :py:data:`scrapy.http.request.NO_CALLBACK` + + .. autodata:: scrapy.http.request.NO_CALLBACK + + For more information, see + :ref:`topics-request-response-ref-request-callback-arguments`. + + .. note:: If exceptions are raised during processing, ``errback`` is + called instead. :type callback: collections.abc.Callable @@ -69,16 +80,24 @@ Request objects 1. Using a dict:: - request_with_cookies = Request(url="http://www.example.com", - cookies={'currency': 'USD', 'country': 'UY'}) + request_with_cookies = Request( + url="http://www.example.com", + cookies={'currency': 'USD', 'country': 'UY'}, + ) 2. Using a list of dicts:: - request_with_cookies = Request(url="http://www.example.com", - cookies=[{'name': 'currency', - 'value': 'USD', - 'domain': 'example.com', - 'path': '/currency'}]) + request_with_cookies = Request( + url="http://www.example.com", + cookies=[ + { + 'name': 'currency', + 'value': 'USD', + 'domain': 'example.com', + 'path': '/currency', + }, + ], + ) The latter form allows for customizing the ``domain`` and ``path`` attributes of the cookie. This is only useful if the cookies are saved diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7bd39aa43..67e14b7b5 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -9,6 +9,7 @@ import logging from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -65,7 +66,8 @@ class RobotsTxtMiddleware: robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, - meta={'dont_obey_robotstxt': True} + meta={'dont_obey_robotstxt': True}, + callback=NO_CALLBACK, ) dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1001fc4a..b57faf121 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -20,6 +20,17 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") +#: When assigned to the ``callback`` parameter of +#: :class:`~scrapy.http.Request`, it indicates that the request it not meant to +#: have a spider callback at all. +#: +#: This value should be used by :ref:`components ` +#: that create and handle their own requests, e.g. through +#: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download +#: middlewares handling such requests can treat them differently from requests +#: intended for the :meth:`~scrapy.Spider.parse` callback. +NO_CALLBACK = object() + class Request(object_ref): """Represents an HTTP request, which is usually generated in a Spider and @@ -63,12 +74,8 @@ class Request(object_ref): raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority - if callback is not None and not callable(callback): - raise TypeError(f'callback must be a callable, got {type(callback).__name__}') - if errback is not None and not callable(errback): - raise TypeError(f'errback must be a callable, got {type(errback).__name__}') - self.callback = callback - self.errback = errback + self._set_xback("callback", callback) + self._set_xback("errback", errback) self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) @@ -78,6 +85,15 @@ class Request(object_ref): self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) + def _set_xback(self, name: str, value: Optional[Callable]) -> None: + if ( + value is not None + and (name != "callback" or value is not NO_CALLBACK) + and not callable(value) + ): + raise TypeError(f'{name} must be a callable, got {type(value).__name__}') + setattr(self, name, value) + @property def cb_kwargs(self) -> dict: if self._cb_kwargs is None: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5308a9793..fc5db58e8 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -7,6 +7,7 @@ from warnings import warn from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result @@ -93,7 +94,7 @@ class MediaPipeline: fp = self._fingerprinter.fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback - request.callback = None + request.callback = NO_CALLBACK request.errback = None # Return cached result if request was already seen diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1460d88eb..71d53ff1a 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware, logger as mw_module_logger) from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from tests.test_robotstxt_interface import rerp_available, reppy_available @@ -53,6 +54,7 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self._get_successful_crawler()) return DeferredList([ self.assertNotIgnored(Request('http://site.local/allowed'), middleware), + maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), @@ -183,6 +185,12 @@ Disallow: /some/randome/page.html return self.assertFailure(maybeDeferred(middleware.process_request, request, spider), IgnoreRequest) + def assertRobotsTxtRequested(self, base_url): + calls = self.crawler.engine.download.call_args_list + request = calls[0][0][0] + self.assertEqual(request.url, f"{base_url}/robots.txt") + self.assertEqual(request.callback, NO_CALLBACK) + class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): if not rerp_available(): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81cebdc7b..e14f8c8f4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,6 +7,7 @@ from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse +from scrapy.http.request import NO_CALLBACK from scrapy.utils.python import to_bytes, to_unicode @@ -277,6 +278,12 @@ class RequestTest(unittest.TestCase): self.assertIs(r4.callback, a_function) self.assertIs(r4.errback, a_function) + r5 = self.request_class( + url='http://example.com', + callback=NO_CALLBACK, + ) + self.assertIs(r5.callback, NO_CALLBACK) + def test_callback_and_errback_type(self): with self.assertRaises(TypeError): self.request_class('http://example.com', callback='a_function') @@ -288,6 +295,8 @@ class RequestTest(unittest.TestCase): callback='a_function', errback='a_function', ) + with self.assertRaises(TypeError): + self.request_class('http://example.com', errback=NO_CALLBACK) def test_from_curl(self): # Note: more curated tests regarding curl conversion are in diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..83572e74f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -33,10 +33,7 @@ from scrapy.utils.test import ( skip_if_no_boto, ) - -def _mocked_download_func(request, info): - response = request.meta.get('response') - return response() if callable(response) else response +from .test_pipeline_media import _mocked_download_func class FilesPipelineTestCase(unittest.TestCase): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6f5466191..3b39212bd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -31,18 +31,13 @@ else: skip_pillow = None -def _mocked_download_func(request, info): - response = request.meta.get('response') - return response() if callable(response) else response - - class ImagesPipelineTestCase(unittest.TestCase): skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() - self.pipeline = ImagesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline = ImagesPipeline(self.tempdir) def tearDown(self): rmtree(self.tempdir) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0a94ae699..99fb424f4 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -9,6 +9,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import signals from scrapy.http import Request, Response +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.pipelines.files import FileException @@ -29,6 +30,7 @@ else: def _mocked_download_func(request, info): + assert request.callback is NO_CALLBACK response = request.meta.get('response') return response() if callable(response) else response From a49346494201d3d7a9d017f16fd64fa2d9042b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 19:53:53 +0100 Subject: [PATCH 02/33] Update the screenshot pipeline code example --- docs/topics/item-pipeline.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 1672ccbcc..fa19d2f4c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -191,6 +191,7 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.http.request import NO_CALLBACK from scrapy.utils.defer import maybe_deferred_to_future @@ -204,8 +205,10 @@ item. adapter = ItemAdapter(item) encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) - request = scrapy.Request(screenshot_url) - response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) + request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) + response = await maybe_deferred_to_future( + spider.crawler.engine.download(request, spider) + ) if response.status != 200: # Error happened, return item. From 5c1559f60e459a9678eecfe24c4fea06e272dbab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 20:30:22 +0100 Subject: [PATCH 03/33] Address typing issues --- scrapy/http/request/__init__.py | 12 ++++++++++-- tox.ini | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index b57faf121..ea73781c8 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,7 +5,8 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from enum import Enum +from typing import Any, Callable, Final, List, Optional, Tuple, Type, TypeVar, Union from w3lib.url import safe_url_string @@ -20,6 +21,11 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") + +# https://github.com/python/typing/issues/689#issuecomment-561425237 +class _NoCallback(Enum): + NO_CALLBACK = 0 + #: When assigned to the ``callback`` parameter of #: :class:`~scrapy.http.Request`, it indicates that the request it not meant to #: have a spider callback at all. @@ -29,7 +35,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") #: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download #: middlewares handling such requests can treat them differently from requests #: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK = object() +NO_CALLBACK: Final = _NoCallback.NO_CALLBACK class Request(object_ref): @@ -49,6 +55,8 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ + callback: Union[None, _NoCallback, Callable] + errback: Optional[Callable] def __init__( self, diff --git a/tox.ini b/tox.ini index 520d90303..076178d8e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py +envlist = security,flake8,typing,py minversion = 1.7.0 [testenv] From 4242ae405d9775098607a67612b5640a4daaf840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 20:37:24 +0100 Subject: [PATCH 04/33] Restore Python 3.7 support --- scrapy/http/request/__init__.py | 4 +++- setup.py | 1 + tox.ini | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ea73781c8..936afb007 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -6,8 +6,9 @@ See documentation in docs/topics/request-response.rst """ import inspect from enum import Enum -from typing import Any, Callable, Final, List, Optional, Tuple, Type, TypeVar, Union +from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing_extensions import Final from w3lib.url import safe_url_string import scrapy @@ -26,6 +27,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class _NoCallback(Enum): NO_CALLBACK = 0 + #: When assigned to the ``callback`` parameter of #: :class:`~scrapy.http.Request`, it indicates that the request it not meant to #: have a spider callback at all. diff --git a/setup.py b/setup.py index bdae28047..049c4f9a6 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ install_requires = [ 'packaging', 'tldextract', 'lxml>=4.3.0', + 'typing-extensions>=3.10.0.0', ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index 076178d8e..1ff5f7a4e 100644 --- a/tox.ini +++ b/tox.ini @@ -94,6 +94,7 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.3.0 + typing-extensions==3.10.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From 818d69fa003a14a2f2058e76258e6269d227efb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 Jan 2023 12:38:07 +0100 Subject: [PATCH 05/33] =?UTF-8?q?Fix=20typo:=20it=20=E2=86=92=20is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrey Rakhmatullin --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 936afb007..302895781 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -29,7 +29,7 @@ class _NoCallback(Enum): #: When assigned to the ``callback`` parameter of -#: :class:`~scrapy.http.Request`, it indicates that the request it not meant to +#: :class:`~scrapy.http.Request`, it indicates that the request is not meant to #: have a spider callback at all. #: #: This value should be used by :ref:`components ` From 0cfe81d1d3978169f6c8ccc90e49315dcd67aca7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 21 Jan 2023 16:57:31 -0300 Subject: [PATCH 06/33] `set-output` command is deprecated --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eee9a4f02..02cf0c9fa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,7 +18,7 @@ jobs: id: check-release-tag run: | if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then - echo ::set-output name=release_tag::true + echo "release_tag=true" >> $GITHUB_OUTPUT fi - name: Publish to PyPI From c883a13006ad98f9f529fe130f7d717ad91f7afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 25 Jan 2023 17:43:10 +0100 Subject: [PATCH 07/33] Make the _set_xback condition more readable --- scrapy/http/request/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a78ba5115..065f2daef 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -106,10 +106,10 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) def _set_xback(self, name: str, value: Optional[Callable]) -> None: - if ( - value is not None - and (name != "callback" or value is not NO_CALLBACK) - and not callable(value) + if not ( + callable(value) + or value is None + or (name == "callback" and value is NO_CALLBACK) ): raise TypeError(f"{name} must be a callable, got {type(value).__name__}") setattr(self, name, value) From 1f3e42897a6697f799d04a4a4320a7c16cbcd939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 25 Jan 2023 18:30:29 +0100 Subject: [PATCH 08/33] =?UTF-8?q?=5FNoCallback=20=E2=86=92=20NoCallbackTyp?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/http/request/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 924bd1d3e..068a4baa3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -24,7 +24,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") # https://github.com/python/typing/issues/689#issuecomment-561425237 -class _NoCallback(Enum): +class NoCallbackType(Enum): NO_CALLBACK = 0 @@ -37,7 +37,7 @@ class _NoCallback(Enum): #: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download #: middlewares handling such requests can treat them differently from requests #: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK: Final = _NoCallback.NO_CALLBACK +NO_CALLBACK: Final = NoCallbackType.NO_CALLBACK class Request(object_ref): @@ -67,7 +67,7 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ - callback: Union[None, _NoCallback, Callable] + callback: Union[None, NoCallbackType, Callable] errback: Optional[Callable] def __init__( From 6d0f9df8c1ad5ad4c4bab4be32cdbe8aed325cd4 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:22:42 -0600 Subject: [PATCH 09/33] added isort.cfg --- .isort.cfg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..a29184f0a --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile = black +multi_line_output = 3 From a5c1ef82762c6c0910abea00c0a6249c40005e44 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:25:15 -0600 Subject: [PATCH 10/33] sort imports with isort --- conftest.py | 1 - docs/_ext/scrapydocs.py | 3 ++- extras/qps-bench-server.py | 7 ++++--- extras/qpsclient.py | 2 +- scrapy/__init__.py | 8 ++++---- scrapy/cmdline.py | 9 +++++---- scrapy/commands/__init__.py | 6 +++--- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 5 +++-- scrapy/commands/edit.py | 2 +- scrapy/commands/fetch.py | 5 +++-- scrapy/commands/genspider.py | 5 ++--- scrapy/commands/parse.py | 6 +++--- scrapy/commands/runspider.py | 6 +++--- scrapy/commands/shell.py | 2 +- scrapy/commands/startproject.py | 7 +++---- scrapy/commands/view.py | 1 + scrapy/contracts/default.py | 2 +- scrapy/core/downloader/__init__.py | 12 +++++------ scrapy/core/downloader/contextfactory.py | 8 ++++---- scrapy/core/downloader/handlers/__init__.py | 1 - scrapy/core/downloader/handlers/http11.py | 6 +++--- scrapy/core/downloader/handlers/http2.py | 1 - scrapy/core/downloader/middleware.py | 2 +- scrapy/core/downloader/tls.py | 4 ++-- scrapy/core/downloader/webclient.py | 6 +++--- scrapy/core/engine.py | 11 +++------- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 5 ++--- scrapy/core/http2/stream.py | 4 ++-- scrapy/core/scheduler.py | 1 - scrapy/core/scraper.py | 6 ++---- scrapy/core/spidermw.py | 5 ++--- scrapy/crawler.py | 6 +++--- scrapy/downloadermiddlewares/ajaxcrawl.py | 3 +-- scrapy/downloadermiddlewares/decompression.py | 1 - scrapy/downloadermiddlewares/httpcache.py | 1 - scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/downloadermiddlewares/retry.py | 3 +-- scrapy/downloadermiddlewares/robotstxt.py | 3 ++- scrapy/dupefilters.py | 3 +-- scrapy/exporters.py | 3 +-- scrapy/extensions/debug.py | 6 +++--- scrapy/extensions/feedexport.py | 5 ++--- scrapy/extensions/httpcache.py | 2 +- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/memusage.py | 6 +++--- scrapy/extensions/statsmailer.py | 2 +- scrapy/extensions/telnet.py | 12 +++++------ scrapy/extensions/throttle.py | 2 +- scrapy/http/__init__.py | 6 ++---- scrapy/http/cookies.py | 4 ++-- scrapy/http/headers.py | 1 + scrapy/http/request/__init__.py | 1 - scrapy/http/request/form.py | 5 ++--- scrapy/http/request/rpc.py | 1 - scrapy/http/response/text.py | 2 +- scrapy/loader/processors.py | 1 - scrapy/logformatter.py | 2 +- scrapy/mail.py | 5 ++--- scrapy/middleware.py | 2 +- scrapy/pipelines/files.py | 1 - scrapy/pipelines/media.py | 4 ++-- scrapy/pqueues.py | 1 - scrapy/resolver.py | 3 +-- scrapy/responsetypes.py | 2 +- scrapy/robotstxt.py | 2 +- scrapy/selector/unified.py | 6 +++--- scrapy/settings/__init__.py | 3 +-- scrapy/shell.py | 7 ++----- scrapy/signalmanager.py | 1 + scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 1 - scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/__init__.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/feed.py | 8 ++++---- scrapy/spiders/sitemap.py | 7 +++---- scrapy/statscollectors.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/conf.py | 1 - scrapy/utils/curl.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/deprecate.py | 3 ++- scrapy/utils/display.py | 1 + scrapy/utils/ftp.py | 2 +- scrapy/utils/httpobj.py | 3 +-- scrapy/utils/iterators.py | 3 +-- scrapy/utils/log.py | 1 - scrapy/utils/misc.py | 9 ++++----- scrapy/utils/ossignal.py | 1 - scrapy/utils/project.py | 6 ++---- scrapy/utils/reqser.py | 1 - scrapy/utils/request.py | 1 - scrapy/utils/response.py | 5 ++--- scrapy/utils/serialize.py | 4 ++-- scrapy/utils/signal.py | 6 ++---- scrapy/utils/spider.py | 1 - scrapy/utils/ssl.py | 2 +- scrapy/utils/template.py | 2 +- scrapy/utils/test.py | 4 ++-- scrapy/utils/testproc.py | 2 +- scrapy/utils/testsite.py | 2 +- scrapy/utils/trackref.py | 1 - scrapy/utils/url.py | 1 + setup.py | 5 +++-- .../asyncio_enabled_reactor_same_loop.py | 3 +-- ..._select_subclass_twisted_reactor_select.py | 1 + tests/CrawlerRunner/ip_address.py | 8 +++++--- tests/keys/__init__.py | 2 +- tests/mockserver.py | 4 ++-- tests/test_closespider.py | 3 ++- tests/test_cmdline/__init__.py | 2 +- .../__init__.py | 2 +- tests/test_command_fetch.py | 4 ++-- tests/test_command_parse.py | 7 ++++--- tests/test_command_shell.py | 7 +++---- tests/test_command_version.py | 3 ++- tests/test_commands.py | 7 +++---- tests/test_contracts.py | 14 ++++++------- tests/test_crawl.py | 2 +- tests/test_crawler.py | 17 +++++++--------- tests/test_downloader_handlers.py | 8 +++++--- tests/test_downloader_handlers_http2.py | 4 ++-- tests/test_downloadermiddleware.py | 8 ++++---- ...test_downloadermiddleware_ajaxcrawlable.py | 3 +-- tests/test_downloadermiddleware_cookies.py | 4 ++-- ...test_downloadermiddleware_decompression.py | 3 ++- ...est_downloadermiddleware_defaultheaders.py | 2 +- ...st_downloadermiddleware_downloadtimeout.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 18 ++++++++--------- ...st_downloadermiddleware_httpcompression.py | 9 +++++---- tests/test_downloadermiddleware_redirect.py | 6 +++--- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 11 +++++----- tests/test_downloadermiddleware_useragent.py | 4 ++-- tests/test_dupefilters.py | 7 ++++--- tests/test_engine.py | 7 +++---- tests/test_engine_stop_download_bytes.py | 5 ++--- tests/test_engine_stop_download_headers.py | 5 ++--- tests/test_exporters.py | 20 +++++++++---------- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 11 +++------- tests/test_http2_client_protocol.py | 13 ++++++------ tests/test_http_cookies.py | 2 +- tests/test_http_headers.py | 2 +- tests/test_http_request.py | 12 +++++------ tests/test_loader.py | 4 ++-- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 4 ++-- tests/test_mail.py | 7 ++++--- tests/test_middleware.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 1 - tests/test_pipeline_media.py | 11 +++++----- tests/test_pipelines.py | 5 ++--- tests/test_pqueues.py | 3 +-- tests/test_proxy_connect.py | 4 ++-- tests/test_request_attribute_binding.py | 5 +---- tests/test_request_cb_kwargs.py | 2 +- tests/test_request_dict.py | 8 +++----- tests/test_request_left.py | 1 + tests/test_responsetypes.py | 4 ++-- tests/test_scheduler.py | 5 ++--- tests/test_selector.py | 2 +- tests/test_settings/__init__.py | 3 ++- tests/test_signals.py | 3 +-- tests/test_spider.py | 10 +++++----- tests/test_spiderloader/__init__.py | 15 +++++++------- tests/test_spidermiddleware.py | 8 ++++---- tests/test_spidermiddleware_depth.py | 2 +- tests/test_spidermiddleware_httperror.py | 10 +++++----- tests/test_spidermiddleware_offsite.py | 6 +++--- tests/test_spidermiddleware_referer.py | 20 +++++++++---------- tests/test_spidermiddleware_urllength.py | 4 ++-- tests/test_spiderstate.py | 5 +++-- tests/test_squeues.py | 9 +++++---- tests/test_squeues_request.py | 16 +++++++-------- tests/test_stats.py | 4 ++-- tests/test_toplevel.py | 4 ++-- tests/test_urlparse_monkeypatches.py | 2 +- tests/test_utils_asyncio.py | 2 +- tests/test_utils_conf.py | 2 +- tests/test_utils_datatypes.py | 1 - tests/test_utils_defer.py | 6 +++--- tests/test_utils_deprecate.py | 2 +- tests/test_utils_display.py | 3 +-- tests/test_utils_gz.py | 3 +-- tests/test_utils_iterators.py | 4 ++-- tests/test_utils_log.py | 8 ++++---- tests/test_utils_misc/__init__.py | 5 ++--- tests/test_utils_project.py | 8 ++++---- tests/test_utils_python.py | 9 ++++----- tests/test_utils_response.py | 9 ++++----- tests/test_utils_serialize.py | 2 +- tests/test_utils_spider.py | 2 +- tests/test_utils_template.py | 4 ++-- tests/test_utils_url.py | 4 ++-- tests/test_webclient.py | 9 +++++---- 202 files changed, 428 insertions(+), 484 deletions(-) diff --git a/conftest.py b/conftest.py index 585356a3e..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,6 @@ import pytest from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor - from tests.keys import generate_keys diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 1419792fc..c23a89089 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,7 +1,8 @@ from operator import itemgetter -from docutils.parsers.rst.roles import set_classes + from docutils import nodes from docutils.parsers.rst import Directive +from docutils.parsers.rst.roles import set_classes from sphinx.util.nodes import make_refnode diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 622164c75..70c9003e5 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -from time import time from collections import deque -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from time import time + from twisted.internet import reactor +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET, Site class Root(Resource): diff --git a/extras/qpsclient.py b/extras/qpsclient.py index bb8527af2..acad71e07 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -8,8 +8,8 @@ usage: """ -from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.spiders import Spider class QPSSpider(Spider): diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0d85198d..44df3d54b 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -8,12 +8,12 @@ import warnings from twisted import version as _txv +from scrapy.http import FormRequest, Request +from scrapy.item import Field, Item +from scrapy.selector import Selector + # Declare top-level shortcuts from scrapy.spiders import Spider -from scrapy.http import Request, FormRequest -from scrapy.selector import Selector -from scrapy.item import Item, Field - __all__ = [ "__version__", diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index ffb40e1c5..730e55350 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,16 +1,17 @@ -import sys -import os import argparse import cProfile import inspect +import os +import sys + import pkg_resources import scrapy +from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules -from scrapy.utils.project import inside_project, get_project_settings +from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index f37d61321..de68c43a5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,16 +1,16 @@ """ Base class for Scrapy commands """ -import os import argparse +import os from pathlib import Path from typing import Any, Dict, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 2e2a21f00..911e5afe6 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,6 @@ +import subprocess import sys import time -import subprocess from urllib.parse import urlencode import scrapy diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index efc7a46ed..de54ca4d3 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,11 +1,12 @@ import time from collections import defaultdict -from unittest import TextTestRunner, TextTestResult as _TextTestResult +from unittest import TextTestResult as _TextTestResult +from unittest import TextTestRunner from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list +from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 537b2013c..ca591011c 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,5 @@ -import sys import os +import sys from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 918db55c6..a9076c5b1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,11 +1,12 @@ import sys + from w3lib.url import is_url from scrapy.commands import ScrapyCommand -from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.http import Request from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index e880e44a9..90dd0874e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,16 +1,15 @@ import os import shutil import string - -from pathlib import Path from importlib import import_module +from pathlib import Path from typing import Optional, cast from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.exceptions import UsageError +from scrapy.utils.template import render_templatefile, string_camelcase def sanitize_module_name(module_name): diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ac97b6193..9c3fc86d4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -2,15 +2,15 @@ import json import logging from typing import Dict -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item +from twisted.internet.defer import maybeDeferred from w3lib.url import is_url -from twisted.internet.defer import maybeDeferred from scrapy.commands import BaseRunSpiderCommand +from scrapy.exceptions import UsageError from scrapy.http import Request from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request -from scrapy.exceptions import UsageError logger = logging.getLogger(__name__) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9751c6c30..8a75f9270 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,13 +1,13 @@ import sys +from importlib import import_module from os import PathLike from pathlib import Path -from importlib import import_module from types import ModuleType from typing import Union -from scrapy.utils.spider import iter_spider_classes -from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand +from scrapy.exceptions import UsageError +from scrapy.utils.spider import iter_spider_classes def _import_file(filepath: Union[str, PathLike]) -> ModuleType: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 1fad8f328..05c76d1eb 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,7 +8,7 @@ from threading import Thread from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell -from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 3ed1f5dbc..88bd5bb33 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,16 +1,15 @@ -import re import os +import re import string from importlib.util import find_spec from pathlib import Path -from shutil import ignore_patterns, move, copy2, copystat +from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.exceptions import UsageError - +from scrapy.utils.template import render_templatefile, string_camelcase TEMPLATES_TO_RENDER = ( ("scrapy.cfg",), diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index a81af7565..ebdfa10a8 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,4 +1,5 @@ import argparse + from scrapy.commands import fetch from scrapy.utils.response import open_in_browser diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index e41d83960..eac702cef 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,6 +1,6 @@ import json -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from scrapy.contracts import Contract from scrapy.exceptions import ContractFail diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3a7de8072..7e0b62bb0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,16 +1,16 @@ import random -from time import time -from datetime import datetime from collections import deque +from datetime import datetime +from time import time from twisted.internet import defer, task +from scrapy import signals +from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.resolver import dnscache from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached -from scrapy.resolver import dnscache -from scrapy import signals -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.core.downloader.handlers import DownloadHandlers class Slot: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 1513638df..53ae78918 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -3,10 +3,10 @@ import warnings from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import ( - optionsForClientTLS, - CertificateOptions, - platformTrust, AcceptableCiphers, + CertificateOptions, + optionsForClientTLS, + platformTrust, ) from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @@ -15,8 +15,8 @@ from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( DEFAULT_CIPHERS, - openssl_methods, ScrapyClientTLSOptions, + openssl_methods, ) from scrapy.utils.misc import create_instance, load_object diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index bb2141d28..39155efe9 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -10,7 +10,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 201c84ff8..8de5459e9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,15 +13,15 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.python.failure import Failure from twisted.web.client import ( + URI, Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, - URI, ) -from twisted.web.http import _DataLoss, PotentialDataLoss +from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from zope.interface import implementer from scrapy import signals diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 25ac0307b..b2579362c 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -16,7 +16,6 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes - H2DownloadHandlerOrSubclass = TypeVar( "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" ) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 3410b4255..5a94e66a6 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -12,8 +12,8 @@ from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.defer import mustbe_deferred, deferred_from_coro from scrapy.utils.conf import build_component_list +from scrapy.utils.defer import deferred_from_coro, mustbe_deferred class DownloaderMiddlewareManager(MiddlewareManager): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index d1c511db0..025575fe1 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -4,12 +4,12 @@ from OpenSSL import SSL from service_identity.exceptions import CertificateError from twisted.internet._sslverify import ( ClientTLSOptions, - verifyHostname, VerificationError, + verifyHostname, ) from twisted.internet.ssl import AcceptableCiphers -from scrapy.utils.ssl import x509name_to_string, get_temp_key_info +from scrapy.utils.ssl import get_temp_key_info, x509name_to_string logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 6421391d0..4558402b2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,15 +1,15 @@ import re from time import time -from urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.http import HTTPClient +from urllib.parse import urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory +from twisted.web.http import HTTPClient from scrapy.http import Headers +from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 19696415b..1efbdb271 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,19 +15,14 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import ( - CloseSpider, - DontCloseSpider, - ScrapyDeprecationWarning, -) -from scrapy.http import Response, Request +from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import BaseSettings from scrapy.spiders import Spider -from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import create_instance, load_object from scrapy.utils.reactor import CallLaterOnce - logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 119443c80..1c43d241c 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -10,7 +10,7 @@ from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpoint from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory -from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.http.request import Request from scrapy.settings import Settings from scrapy.spiders import Spider diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 214deeed0..0bf69e513 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,9 @@ from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, ConnectionTerminated, DataReceived, + Event, ResponseReceived, SettingsAcknowledged, StreamEnded, @@ -23,7 +23,7 @@ from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory -from twisted.internet.protocol import connectionDone, Factory, Protocol +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure @@ -35,7 +35,6 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider - logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 2b5c98e5f..87beb41e5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,13 +1,13 @@ import logging from enum import Enum from io import BytesIO +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from urllib.parse import urlparse -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError from hpack import HeaderTuple -from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1e6fc69e1..3c46e3a5f 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -12,7 +12,6 @@ from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object - logger = logging.getLogger(__name__) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7c2eefbe6..1a09f22f7 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging from collections import deque from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, @@ -13,7 +14,6 @@ from typing import ( Iterable, Optional, Set, - TYPE_CHECKING, Tuple, Union, ) @@ -22,7 +22,7 @@ from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -34,12 +34,10 @@ from scrapy.utils.defer import ( parallel, parallel_async, ) - from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output - if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aaed5865..ba9c37e38 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -28,14 +28,13 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( - mustbe_deferred, - deferred_from_coro, deferred_f_from_coro_f, + deferred_from_coro, maybe_deferred_to_future, + mustbe_deferred, ) from scrapy.utils.python import MutableAsyncChain, MutableChain - logger = logging.getLogger(__name__) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f58cd73d3..397817d6f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -17,20 +17,20 @@ except ImportError: from zope.interface.verify import verifyClass -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader -from scrapy.settings import overridden_settings, Settings +from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import ( + LogCounterHandler, configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, log_reactor_info, log_scrapy_info, - LogCounterHandler, ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 86ff7b9fe..137ed5b18 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,12 +1,11 @@ -import re import logging +import re from w3lib import html from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse - logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 410015281..368ca60f7 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -14,7 +14,6 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes - warn( "scrapy.downloadermiddlewares.decompression is deprecated", ScrapyDeprecationWarning, diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index eb2754f1d..74c55f6e2 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -23,7 +23,6 @@ from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object - HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 489867918..f74d84b69 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,6 +1,6 @@ import base64 from urllib.parse import unquote, urlunparse -from urllib.request import getproxies, proxy_bypass, _parse_proxy +from urllib.request import _parse_proxy, getproxies, proxy_bypass from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 003c59fc4..f442a3012 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -3,10 +3,10 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh -from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8a8f15f9a..11a30911c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,7 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -from logging import getLogger, Logger +from logging import Logger, getLogger from typing import Optional, Union from twisted.internet import defer @@ -31,7 +31,6 @@ from scrapy.spiders import Spider from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message - retry_logger = getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 326c35290..89f8f7428 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. import logging from twisted.internet.defer import Deferred, maybeDeferred -from scrapy.exceptions import NotConfigured, IgnoreRequest + +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index fa0f8f846..d796e5cbb 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -10,8 +10,7 @@ from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import referer_str, RequestFingerprinter - +from scrapy.utils.request import RequestFingerprinter, referer_str BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 7d9a9b6ff..bb3e3c662 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,14 +11,13 @@ import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder - __all__ = [ "BaseItemExporter", "PprintItemExporter", diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 8628b4a1e..1b6c7777f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,11 +4,11 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ -import sys -import signal import logging -import traceback +import signal +import sys import threading +import traceback from pdb import Pdb from scrapy.utils.engine import format_engine_status diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 823955aa3..cd26b5778 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -16,9 +16,9 @@ from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path -from zope.interface import implementer, Interface +from zope.interface import Interface, implementer -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available @@ -28,7 +28,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import get_func_args, without_none_values - logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index bbddaac40..2d120a6ed 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -7,7 +7,7 @@ from pathlib import Path from time import time from weakref import WeakKeyDictionary -from w3lib.http import headers_raw_to_dict, headers_dict_to_raw +from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 6295dcdb7..78874a6db 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -2,8 +2,8 @@ import logging from twisted.internet import task -from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 2bba71972..221967bda 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,11 +3,11 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ -import sys -import socket import logging -from pprint import pformat +import socket +import sys from importlib import import_module +from pprint import pformat from twisted.internet import task diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 8733ad22b..58610c25e 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -5,8 +5,8 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ from scrapy import signals -from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured +from scrapy.mail import MailSender class StatsMailer: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 271f22428..c92b7f5fe 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,11 +4,11 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ -import pprint -import logging -import traceback import binascii +import logging import os +import pprint +import traceback from twisted.internet import protocol @@ -21,12 +21,12 @@ except (ImportError, SyntaxError): _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.trackref import print_live_refs +from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp -from scrapy.utils.decorators import defers +from scrapy.utils.trackref import print_live_refs logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 79e20de2a..396800775 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ import logging -from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index e6c58e1f1..ac3946302 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -6,13 +6,11 @@ Request and Response outside this module. """ from scrapy.http.headers import Headers - from scrapy.http.request import Request from scrapy.http.request.form import FormRequest -from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.request.json_request import JsonRequest - +from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse -from scrapy.http.response.xml import XmlResponse from scrapy.http.response.text import TextResponse +from scrapy.http.response.xml import XmlResponse diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 94afedb08..a5329ad51 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,11 +1,11 @@ import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy +from http.cookiejar import CookieJar as _CookieJar +from http.cookiejar import DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode - # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index a5db30d6f..2540be01a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from w3lib.http import headers_dict_to_raw + from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e290f2143..0e925301e 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -17,7 +17,6 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax - RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 993219745..bdc6a3e39 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union -from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit +from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node @@ -14,10 +14,9 @@ from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse -from scrapy.utils.python import to_bytes, is_listlike +from scrapy.utils.python import is_listlike, to_bytes from scrapy.utils.response import get_base_url - FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c0a6e86c1..43692923b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -10,7 +10,6 @@ from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args - DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f9df4e1b0..e45d95602 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -15,8 +15,8 @@ from w3lib.encoding import ( html_body_declared_encoding, html_to_unicode, http_content_type_encoding, - resolve_encoding, read_bom, + resolve_encoding, ) from w3lib.html import strip_html5_whitespace diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index f27a669d6..b82c6d5c7 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -7,7 +7,6 @@ from itemloaders import processors from scrapy.utils.deprecate import create_deprecated_class - MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) Compose = create_deprecated_class("Compose", processors.Compose) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index e0b93d812..560006c95 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,5 +1,5 @@ -import os import logging +import os from twisted.python.failure import Failure diff --git a/scrapy/mail.py b/scrapy/mail.py index fa1e55f1f..43115c53e 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,14 +12,13 @@ from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from twisted.python.versions import Version -from twisted.internet import defer, ssl from twisted import version as twisted_version +from twisted.internet import defer, ssl +from twisted.python.versions import Version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes - logger = logging.getLogger(__name__) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 15f5b23e0..f82d722fa 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -8,8 +8,8 @@ from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings +from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 01a9c41fe..d925fc984 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -32,7 +32,6 @@ from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str - logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index af23b4cc8..f6eb5b139 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -9,10 +9,10 @@ from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import arg_to_iter logger = logging.getLogger(__name__) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 6f65184e5..62a9af477 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,7 +3,6 @@ import logging from scrapy.utils.misc import create_instance - logger = logging.getLogger(__name__) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f5d2b8e05..6cbe01cbf 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( - IHostResolution, IHostnameResolver, + IHostResolution, IResolutionReceiver, IResolverSimple, ) @@ -10,7 +10,6 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache - # TODO: cache misses dnscache = LocalCache(10000) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6b489bd8b..6af8915c2 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,9 +2,9 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ +from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from io import StringIO from scrapy.http import Response from scrapy.utils.misc import load_object diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0dadeef92..604b5e314 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,5 +1,5 @@ -import sys import logging +import sys from abc import ABCMeta, abstractmethod from scrapy.utils.python import to_unicode diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 6ba87428e..cff97104a 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -3,10 +3,10 @@ XPath selectors based on lxml """ from parsel import Selector as _ParselSelector -from scrapy.utils.trackref import object_ref -from scrapy.utils.python import to_bytes -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, XmlResponse +from scrapy.utils.python import to_bytes +from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c0d0741c5..fde8fdde4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,12 +1,11 @@ -import json import copy +import json from collections.abc import MutableMapping from importlib import import_module from pprint import pformat from scrapy.settings import default_settings - SETTINGS_PRIORITIES = { "default": 0, "command": 10, diff --git a/scrapy/shell.py b/scrapy/shell.py index 084a27141..ae6e641fd 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,7 +7,7 @@ import os import signal from itemadapter import is_item -from twisted.internet import threads, defer +from twisted.internet import defer, threads from twisted.python import threadable from w3lib.url import any_to_uri @@ -20,11 +20,8 @@ from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser -from scrapy.utils.reactor import ( - is_asyncio_reactor_installed, - set_asyncio_event_loop, -) class Shell: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f00447a55..d7e3bce91 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,4 +1,5 @@ from pydispatch import dispatcher + from scrapy.utils import signal as _signal diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index c57ec8d48..1a48926b3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,8 +3,8 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import re import logging +import re import warnings from scrapy import signals diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a99b6315b..d86f55a40 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -15,7 +15,6 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url - LOCAL_SCHEMES = ( "about", "blob", diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 9a21379f9..f6d92e53a 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -6,8 +6,8 @@ See documentation in docs/topics/spider-middleware.rst import logging -from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.http import Request logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index f8cac5458..3502f8b27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -97,5 +97,5 @@ class Spider(object_ref): # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider from scrapy.spiders.sitemap import SitemapSpider diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d75b455ae..05c425948 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst import copy from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, Response, HtmlResponse +from scrapy.http import HtmlResponse, Request, Response from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index b3c5ff01e..5ec0504a8 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,11 +4,11 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spiders import Spider -from scrapy.utils.iterators import xmliter, csviter -from scrapy.utils.spider import iterate_spider_output -from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.selector import Selector +from scrapy.spiders import Spider +from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.spider import iterate_spider_output class XMLFeedSpider(Spider): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index a1734a3b1..c3cca9699 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,11 +1,10 @@ -import re import logging +import re -from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse -from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip, gzip_magic_number - +from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots logger = logging.getLogger(__name__) diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 4181c7a2f..dd3c32737 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,8 +1,8 @@ """ Scrapy extension for collecting scraping stats """ -import pprint import logging +import pprint logger = logging.getLogger(__name__) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 32bc2e38c..1089ba7b8 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,8 +1,8 @@ import random from urllib.parse import urlencode -from twisted.web.server import Site from twisted.web.resource import Resource +from twisted.web.server import Site class Root(Resource): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 2f1569ab6..3ade1d105 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError - from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 3175e5fdc..a2243ae2e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,7 +1,7 @@ import argparse import warnings -from shlex import split from http.cookies import SimpleCookie +from shlex import split from urllib.parse import urlparse from w3lib.http import basic_auth_header diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21cd5e78f..ec130d685 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop +from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed def defer_fail(_failure: Failure) -> Deferred: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 4757fef0a..61a4347ea 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,8 +1,9 @@ """Some helpers for deprecation messages""" -import warnings import inspect +import warnings from typing import List, Tuple + from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6dceb87f..77c32b002 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,6 +6,7 @@ import ctypes import platform import sys from pprint import pformat as pformat_ + from packaging.version import Version as parse_version diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 9dbb4180f..6bf6e9195 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,5 +1,5 @@ import posixpath -from ftplib import error_perm, FTP +from ftplib import FTP, error_perm from posixpath import dirname diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 540035ca9..d502e8910 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,11 @@ """Helper functions for scrapy.http objects (Request, Response)""" from typing import Union -from urllib.parse import urlparse, ParseResult +from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary from scrapy.http import Request, Response - _urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7d52d35c9..170055d5e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,11 +3,10 @@ import logging import re from io import StringIO -from scrapy.http import TextResponse, Response +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2560a421f..6ae27dc29 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -11,7 +11,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index dfd2f767c..f9f9c0d5b 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,23 +1,22 @@ """Helper functions which don't fit anywhere else""" import ast +import hashlib import inspect import os import re -import hashlib import warnings from collections import deque from contextlib import contextmanager +from functools import partial from importlib import import_module from pkgutil import iter_modules -from functools import partial from w3lib.html import replace_entities -from scrapy.utils.datatypes import LocalWeakReferencedCache -from scrapy.utils.python import flatten, to_unicode from scrapy.item import Item +from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.deprecate import ScrapyDeprecationWarning - +from scrapy.utils.python import flatten, to_unicode _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 18d856927..7646264a8 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,6 +1,5 @@ import signal - signal_names = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 4fbb6bcaf..ab1b8e3ee 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,13 +1,11 @@ import os import warnings - from importlib import import_module from pathlib import Path -from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env -from scrapy.settings import Settings from scrapy.exceptions import NotConfigured - +from scrapy.settings import Settings +from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c818c8700..15705db83 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -5,7 +5,6 @@ import scrapy from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.request import request_from_dict as _from_dict - warnings.warn( ( "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 3e29a9c57..409ca2e52 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -19,7 +19,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode - _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a91a49170..730d005e8 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -11,12 +11,11 @@ from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html + import scrapy from scrapy.http.response import Response - - -from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.decorators import deprecated +from scrapy.utils.python import to_bytes, to_unicode _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 3602043f3..358f41679 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,8 +1,8 @@ -import json import datetime import decimal +import json -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from twisted.internet import defer from scrapy.http import Request, Response diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index b7c284174..b95786d35 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,9 +2,6 @@ import collections.abc import logging -from twisted.internet.defer import DeferredList, Deferred -from twisted.python.failure import Failure - from pydispatch.dispatcher import ( Anonymous, Any, @@ -13,12 +10,13 @@ from pydispatch.dispatcher import ( liveReceivers, ) from pydispatch.robustapply import robustApply +from twisted.internet.defer import Deferred, DeferredList +from twisted.python.failure import Failure from scrapy.exceptions import StopDownload from scrapy.utils.defer import maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f829bceb8..86449eeb2 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -5,7 +5,6 @@ from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 9f03621c1..f4b598ac7 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,5 @@ -import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil +import OpenSSL.SSL from scrapy.utils.python import to_unicode diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 89bedfc69..1499aeb3d 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,8 +1,8 @@ """Helper functions for working with templates""" -from os import PathLike import re import string +from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d21065706..58576903a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,11 +4,11 @@ This module contains some assorted functions used in tests import asyncio import os +from importlib import import_module from pathlib import Path from posixpath import split from unittest import mock -from importlib import import_module from twisted.trial.unittest import SkipTest from scrapy.utils.boto import is_botocore_available @@ -109,7 +109,7 @@ def mock_google_cloud_storage(): """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ - from google.cloud.storage import Client, Bucket, Blob + from google.cloud.storage import Blob, Bucket, Client client_mock = mock.create_autospec(Client) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index fe5c8d74c..ecb2e31bf 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,5 +1,5 @@ -import sys import os +import sys from twisted.internet import defer, protocol diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index a47756c4b..de9ce992a 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,6 +1,6 @@ from urllib.parse import urljoin -from twisted.web import server, resource, static, util +from twisted.web import resource, server, static, util class SiteTest: diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9aa775a1b..01b980c93 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -15,7 +15,6 @@ from time import time from typing import DefaultDict from weakref import WeakKeyDictionary - NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index cd8a6a05a..0a27ccd6d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -12,6 +12,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # move doesn't break old code from w3lib.url import * from w3lib.url import _safe_chars, _unquotepath # noqa: F401 + from scrapy.utils.python import to_unicode diff --git a/setup.py b/setup.py index f53334d4e..c6bcf2439 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ from pathlib import Path -from pkg_resources import parse_version -from setuptools import setup, find_packages, __version__ as setuptools_version +from pkg_resources import parse_version +from setuptools import __version__ as setuptools_version +from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index 79dd77bb2..be9c83b95 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -1,9 +1,8 @@ import asyncio import sys -from uvloop import Loop - from twisted.internet import asyncioreactor +from uvloop import Loop if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 37626c081..a8f707841 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,5 +1,6 @@ from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor + import scrapy from scrapy.crawler import CrawlerProcess diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 26db16dd6..23260ab0d 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,14 +1,16 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names import cache +from twisted.names import hosts as hostsModule +from twisted.names import resolve from twisted.names.client import Resolver from twisted.python.runtime import platform -from scrapy import Spider, Request +from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging -from tests.mockserver import MockServer, MockDNSServer +from tests.mockserver import MockDNSServer, MockServer # https://stackoverflow.com/a/32784190 diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index b306437db..5cc65a903 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -14,8 +14,8 @@ from cryptography.x509 import ( DNSName, Name, NameAttribute, - random_serial_number, SubjectAlternativeName, + random_serial_number, ) from cryptography.x509.oid import NameOID diff --git a/tests/mockserver.py b/tests/mockserver.py index e07ae8797..7991da9dc 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -4,7 +4,7 @@ import random import sys from pathlib import Path from shutil import rmtree -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from tempfile import mkdtemp from urllib.parse import urlencode @@ -14,7 +14,7 @@ from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory from twisted.web import resource, server -from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site +from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site from twisted.web.static import File from twisted.web.util import redirectTo diff --git a/tests/test_closespider.py b/tests/test_closespider.py index c497450f7..9b39187d5 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,8 +1,9 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.utils.test import get_crawler -from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider from tests.mockserver import MockServer +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider class TestCloseSpider(TestCase): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 5aa35a6d9..15833cd19 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -6,7 +6,7 @@ import tempfile import unittest from io import StringIO from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from scrapy.utils.test import get_testenv diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index d5088e817..5cb09b5c0 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,7 +1,7 @@ import sys import unittest from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen class CmdlineCrawlPipelineTest(unittest.TestCase): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index bd44fa76e..124c968c2 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,8 +1,8 @@ -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest -from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from scrapy.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 1ee1bf5a7..b0fb978e9 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,13 +1,14 @@ -import os import argparse +import os from pathlib import Path from twisted.internet import defer + from scrapy.commands import parse from scrapy.settings import Settings -from scrapy.utils.testsite import SiteTest -from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.testsite import SiteTest from tests.test_commands import CommandTest diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7e99a7296..8ce82db86 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,12 +1,11 @@ from pathlib import Path -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest -from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest - -from tests import tests_datadir, NON_EXISTING_RESOLVABLE +from scrapy.utils.testsite import SiteTest +from tests import NON_EXISTING_RESOLVABLE, tests_datadir class ShellTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_version.py b/tests/test_command_version.py index f97a088a8..3bf6019b5 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,6 +1,7 @@ import sys -from twisted.trial import unittest + from twisted.internet import defer +from twisted.trial import unittest import scrapy from scrapy.utils.testproc import ProcessTest diff --git a/tests/test_commands.py b/tests/test_commands.py index 363e87aa7..5ff2dd482 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,6 +1,6 @@ +import argparse import inspect import json -import argparse import os import platform import re @@ -10,7 +10,7 @@ import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path -from shutil import rmtree, copytree +from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer @@ -23,12 +23,11 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv - from tests.test_crawler import ExceptionSpider, NoRequestsSpider diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 7b104f618..813927fc5 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -5,18 +5,18 @@ from twisted.python import failure from twisted.trial import unittest from scrapy import FormRequest -from scrapy.spidermiddlewares.httperror import HttpError -from scrapy.spiders import Spider -from scrapy.http import Request -from scrapy.item import Item, Field -from scrapy.utils.test import get_crawler -from scrapy.contracts import ContractsManager, Contract +from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( - UrlContract, CallbackKeywordArgumentsContract, ReturnsContract, ScrapesContract, + UrlContract, ) +from scrapy.http import Request +from scrapy.item import Field, Item +from scrapy.spidermiddlewares.httperror import HttpError +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4139f1b11..ca9084294 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,9 +1,9 @@ import json import logging +import unittest from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse -import unittest from pytest import mark from testfixtures import LogCapture diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c6b93599e..706bfbaa9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -5,28 +5,25 @@ import sys import warnings from pathlib import Path -from pytest import raises, mark +from pkg_resources import parse_version +from pytest import mark, raises from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version from twisted.trial import unittest - -from pkg_resources import parse_version from w3lib import __version__ as w3lib_version import scrapy -from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.extensions import telnet +from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler -from scrapy.extensions.throttle import AutoThrottle -from scrapy.extensions import telnet -from scrapy.utils.test import get_testenv - +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler, get_testenv from tests.mockserver import MockServer diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4f953439d..fd4176e2f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -5,7 +5,7 @@ import sys import tempfile from pathlib import Path from typing import Optional, Type -from unittest import mock, SkipTest +from unittest import SkipTest, mock from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal @@ -1041,7 +1041,8 @@ class BaseFTPTestCase(unittest.TestCase): ) def setUp(self): - from twisted.protocols.ftp import FTPRealm, FTPFactory + from twisted.protocols.ftp import FTPFactory, FTPRealm + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file @@ -1190,7 +1191,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): req_meta = {} def setUp(self): - from twisted.protocols.ftp import FTPRealm, FTPFactory + from twisted.protocols.ftp import FTPFactory, FTPRealm + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index fd765089a..8090d50b9 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -15,10 +15,10 @@ from scrapy.utils.misc import create_instance from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( - Https11TestCase, - Https11CustomCiphers, Http11MockServerTestCase, Http11ProxyTestCase, + Https11CustomCiphers, + Https11TestCase, UriResource, ) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index d8e377519..2be32e37b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -4,15 +4,15 @@ from unittest import mock from pytest import mark from twisted.internet import defer from twisted.internet.defer import Deferred -from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.trial.unittest import TestCase +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider -from scrapy.exceptions import _InvalidOutput -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.utils.test import get_crawler, get_from_asyncio_queue from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler, get_from_asyncio_queue class ManagerTestCase(TestCase): diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 6be107f6f..043dc0a12 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,11 +1,10 @@ import unittest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware +from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider -from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler - __doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 812c003da..4a81a638e 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,14 +1,14 @@ import logging from unittest import TestCase -from testfixtures import LogCapture import pytest +from testfixtures import LogCapture from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 16ae9ed75..412c20a78 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -1,6 +1,7 @@ from unittest import TestCase, main -from scrapy.http import Response, XmlResponse + from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware +from scrapy.http import Response, XmlResponse from scrapy.spiders import Spider from scrapy.utils.test import assert_samelines from tests import get_testdata diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 601e85799..27d6224b4 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -3,8 +3,8 @@ from unittest import TestCase from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler class TestDefaultHeadersMiddleware(TestCase): diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 8d2b821b0..44458ade8 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,8 +1,8 @@ import unittest from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware -from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 1320bded2..6b79234d0 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -3,9 +3,9 @@ import unittest import pytest from w3lib.http import basic_auth_header +from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index caa89b6bd..a355a9b5b 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,16 +1,16 @@ -import time -import tempfile -import shutil -import unittest import email.utils +import shutil +import tempfile +import time +import unittest from contextlib import contextmanager -from scrapy.http import Response, HtmlResponse, Request -from scrapy.spiders import Spider -from scrapy.settings import Settings -from scrapy.exceptions import IgnoreRequest -from scrapy.utils.test import get_crawler from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import HtmlResponse, Request, Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index efae7c4e0..fac5588ff 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,18 +1,19 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path -from unittest import TestCase, SkipTest +from unittest import SkipTest, TestCase from warnings import catch_warnings from w3lib.encoding import resolve_encoding -from scrapy.spiders import Spider -from scrapy.http import Response, Request, HtmlResponse + from scrapy.downloadermiddlewares.httpcompression import ( - HttpCompressionMiddleware, ACCEPTED_ENCODINGS, + HttpCompressionMiddleware, ) from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e2ff9ec2b..dc15b672c 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,12 +1,12 @@ import unittest from scrapy.downloadermiddlewares.redirect import ( - RedirectMiddleware, MetaRefreshMiddleware, + RedirectMiddleware, ) -from scrapy.spiders import Spider from scrapy.exceptions import IgnoreRequest -from scrapy.http import Request, Response, HtmlResponse +from scrapy.http import HtmlResponse, Request, Response +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cadd647ad..02854c2a7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -12,7 +12,7 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware +from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.spiders import Spider diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ac08c6006..f98e0b12e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,17 +1,16 @@ from unittest import mock -from twisted.internet import reactor, error +from twisted.internet import error, reactor from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure from twisted.trial import unittest -from scrapy.downloadermiddlewares.robotstxt import ( - RobotsTxtMiddleware, - logger as mw_module_logger, -) + +from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware +from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings -from tests.test_robotstxt_interface import rerp_available, reppy_available +from tests.test_robotstxt_interface import reppy_available, rerp_available class RobotsTxtMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 0702dd042..cad3dea5c 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,8 +1,8 @@ from unittest import TestCase -from scrapy.spiders import Spider -from scrapy.http import Request from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware +from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 4019012d1..aa0975555 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,14 +1,15 @@ import hashlib -import tempfile -import unittest import shutil import sys +import tempfile +import unittest from pathlib import Path + from testfixtures import LogCapture +from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request -from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider diff --git a/tests/test_engine.py b/tests/test_engine.py index 7ddb420ba..02b59f448 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -14,13 +14,13 @@ import re import subprocess import sys from collections import defaultdict +from dataclasses import dataclass from pathlib import Path from threading import Timer from urllib.parse import urlparse -from dataclasses import dataclass -import pytest import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -31,12 +31,11 @@ from scrapy import signals from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler - from tests import get_testdata, tests_datadir diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index fb8dd4313..8dbb5b7ea 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -2,14 +2,13 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.exceptions import StopDownload - from tests.test_engine import ( AttrsItemsSpider, + CrawlerRun, DataClassItemsSpider, DictItemsSpider, - TestSpider, - CrawlerRun, EngineTest, + TestSpider, ) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 93437559d..0bad5ba55 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -2,14 +2,13 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.exceptions import StopDownload - from tests.test_engine import ( AttrsItemsSpider, + CrawlerRun, DataClassItemsSpider, DictItemsSpider, - TestSpider, - CrawlerRun, EngineTest, + TestSpider, ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 7689045b7..8e0999348 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,31 +1,31 @@ -import re +import dataclasses import json import marshal import pickle +import re import tempfile import unittest -import dataclasses -from io import BytesIO from datetime import datetime +from io import BytesIO from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter -from scrapy.item import Item, Field -from scrapy.utils.python import to_unicode from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, - PprintItemExporter, - PickleItemExporter, CsvItemExporter, - XmlItemExporter, - JsonLinesItemExporter, JsonItemExporter, - PythonItemExporter, + JsonLinesItemExporter, MarshalItemExporter, + PickleItemExporter, + PprintItemExporter, + PythonItemExporter, + XmlItemExporter, ) +from scrapy.item import Field, Item +from scrapy.utils.python import to_unicode def custom_serializer(value): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index e36c45d8e..9fd680e9f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,7 +1,7 @@ -from twisted.trial import unittest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from twisted.internet import defer +from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 09a4aa823..96f97ca99 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -19,7 +19,7 @@ from pathlib import Path from string import ascii_letters, digits from typing import Union from unittest import mock -from urllib.parse import urljoin, quote +from urllib.parse import quote, urljoin from urllib.request import pathname2url import lxml.etree @@ -35,7 +35,6 @@ import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( - _FeedSlot, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -44,15 +43,11 @@ from scrapy.extensions.feedexport import ( IFeedStorage, S3FeedStorage, StdoutFeedStorage, + _FeedSlot, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import ( - get_crawler, - mock_google_cloud_storage, - skip_if_no_boto, -) - +from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 88345d2bc..17a94f036 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -17,18 +17,19 @@ from twisted.internet.defer import ( ) from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate +from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from twisted.web.client import ResponseFailed, URI -from twisted.web.http import H2_ENABLED, Request as TxRequest -from twisted.web.server import Site, NOT_DONE_YET +from twisted.web.client import URI, ResponseFailed +from twisted.web.http import H2_ENABLED +from twisted.web.http import Request as TxRequest +from twisted.web.server import NOT_DONE_YET, Site from twisted.web.static import File -from scrapy.http import Request, Response, JsonRequest +from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider -from tests.mockserver import ssl_context_factory, LeafResource, Status +from tests.mockserver import LeafResource, Status, ssl_context_factory def generate_random_string(size): diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index ea42cadcd..9e43b72b0 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,5 +1,5 @@ -from urllib.parse import urlparse from unittest import TestCase +from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 566bb302d..7db1eb8c5 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,5 +1,5 @@ -import unittest import copy +import unittest from scrapy.http import Headers diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0c10b27a0..d02f11f0e 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,18 +1,18 @@ -import unittest -import re import json -import xmlrpc.client +import re +import unittest import warnings +import xmlrpc.client from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import ( - Request, FormRequest, - XmlRpcRequest, - JsonRequest, Headers, HtmlResponse, + JsonRequest, + Request, + XmlRpcRequest, ) from scrapy.utils.python import to_bytes, to_unicode diff --git a/tests/test_loader.py b/tests/test_loader.py index 9dd298864..5f4750ff3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,12 +1,12 @@ -import unittest import dataclasses +import unittest import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst from scrapy.http import HtmlResponse, Response -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.loader import ItemLoader from scrapy.selector import Selector diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 8757db0ce..638af825b 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -16,7 +16,7 @@ from itemloaders.processors import ( TakeFirst, ) -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.loader import ItemLoader from scrapy.loader.common import wrap_loader_context from scrapy.utils.deprecate import ScrapyDeprecationWarning diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 11cf6d81a..0971a5a38 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -6,11 +6,11 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.exceptions import DropItem -from scrapy.utils.test import get_crawler from scrapy.http import Request, Response -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.logformatter import LogFormatter from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import ItemSpider diff --git a/tests/test_mail.py b/tests/test_mail.py index 0ee0400cd..bc7298e9d 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,14 +1,15 @@ # coding=utf-8 import unittest -from io import BytesIO from email.charset import Charset +from io import BytesIO +from twisted import version as twisted_version +from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions from twisted.internet.ssl import ClientContextFactory from twisted.python.versions import Version -from twisted.internet import defer -from twisted import version as twisted_version + from scrapy.mail import MailSender diff --git a/tests/test_middleware.py b/tests/test_middleware.py index a84cf4c28..00ff746ee 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,8 +1,8 @@ from twisted.trial import unittest -from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings class M1: diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index b04da22be..8f5d87ebf 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -6,8 +6,8 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy.crawler import CrawlerRunner from scrapy import signals +from scrapy.crawler import CrawlerRunner from tests.mockserver import MockServer from tests.spiders import SimpleSpider diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 43942e53e..13de042a4 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,3 +1,4 @@ +import dataclasses import os import random import time @@ -8,7 +9,6 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import mock from urllib.parse import urlparse -import dataclasses import attr from itemadapter import ItemAdapter diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 38a2d6c41..a5a495393 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -18,7 +18,6 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes - try: from PIL import Image except ImportError: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 87ab03395..e6d8ed2a2 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,25 +1,24 @@ -from typing import Optional import io +from typing import Optional from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.python.failure import Failure +from twisted.trial import unittest from scrapy import signals from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline +from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler - try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 7b905d321..5ab288c1a 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -5,10 +5,9 @@ from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial import unittest -from scrapy import Spider, signals, Request -from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future +from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue - from tests.mockserver import MockServer diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 96a64c19d..1584014b8 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -4,11 +4,10 @@ import unittest import queuelib from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue from scrapy.utils.test import get_crawler - from tests.test_scheduler import MockDownloader, MockEngine diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 5aeae7546..c05f4da91 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -3,15 +3,15 @@ import os import re import sys from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from urllib.parse import urlsplit, urlunsplit + from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer from tests.spiders import SimpleSpider, SingleRequestSpider diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 17c0309d1..d65d74206 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,16 +1,13 @@ +from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -from testfixtures import LogCapture - from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider - OVERRIDDEN_URL = "https://example.org" diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 454b68942..577522c6c 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -4,8 +4,8 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.utils.test import get_crawler -from tests.spiders import MockServerSpider from tests.mockserver import MockServer +from tests.spiders import MockServerSpider class InjectArgumentsDownloaderMiddleware: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index d9067610e..8665a9205 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -3,7 +3,7 @@ import unittest import warnings from contextlib import suppress -from scrapy import Spider, Request +from scrapy import Request, Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -171,10 +171,8 @@ class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): "scrapy.utils.reqser" ] # delete module to reset the deprecation warning - from scrapy.utils.reqser import ( - request_from_dict as _from_dict, - request_to_dict as _to_dict, - ) + from scrapy.utils.reqser import request_from_dict as _from_dict + from scrapy.utils.reqser import request_to_dict as _to_dict request_copy = _from_dict(_to_dict(request, spider), spider) self._assert_same_request(request, request_copy) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index d08ed0f68..54155f7ef 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,5 +1,6 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 57484a2a1..859960518 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,7 +1,7 @@ import unittest -from scrapy.responsetypes import responsetypes -from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers +from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse +from scrapy.responsetypes import responsetypes class ResponseTypesTest(unittest.TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 67728321d..5acc412e5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,21 +1,20 @@ +import collections import shutil import tempfile import unittest -import collections from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.crawler import Crawler from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler +from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler from tests.mockserver import MockServer - MockEngine = collections.namedtuple("MockEngine", ["downloader"]) MockSlot = collections.namedtuple("MockSlot", ["active"]) diff --git a/tests/test_selector.py b/tests/test_selector.py index ad72e068d..febae46ac 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -2,7 +2,7 @@ import weakref from twisted.trial import unittest -from scrapy.http import TextResponse, HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9a01fd433..2a3b2d529 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -2,12 +2,13 @@ import unittest from unittest import mock from scrapy.settings import ( + SETTINGS_PRIORITIES, BaseSettings, Settings, SettingsAttribute, - SETTINGS_PRIORITIES, get_settings_priority, ) + from . import default_settings diff --git a/tests/test_signals.py b/tests/test_signals.py index 4c6ffabdc..0df104600 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -2,9 +2,8 @@ from pytest import mark from twisted.internet import defer from twisted.trial import unittest -from scrapy import signals, Request, Spider +from scrapy import Request, Spider, signals from scrapy.utils.test import get_crawler, get_from_asyncio_queue - from tests.mockserver import MockServer diff --git a/tests/test_spider.py b/tests/test_spider.py index 540091516..eb8a1f9f0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -6,21 +6,21 @@ from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest - from w3lib.url import safe_url_string + from scrapy import signals +from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse +from scrapy.linkextractors import LinkExtractor from scrapy.settings import Settings -from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse -from scrapy.spiders.init import InitSpider from scrapy.spiders import ( - CSVFeedSpider, CrawlSpider, + CSVFeedSpider, Rule, SitemapSpider, Spider, XMLFeedSpider, ) -from scrapy.linkextractors import LinkExtractor +from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler from tests import get_testdata diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3745355a0..da656303d 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,21 +1,20 @@ -import sys import shutil +import sys +import tempfile import warnings from pathlib import Path -import tempfile -from zope.interface.verify import verifyObject from twisted.trial import unittest - +from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy -from scrapy.interfaces import ISpiderLoader -from scrapy.spiderloader import SpiderLoader -from scrapy.settings import Settings -from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.interfaces import ISpiderLoader +from scrapy.settings import Settings +from scrapy.spiderloader import SpiderLoader module_dir = Path(__file__).resolve().parent diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 760ee43df..974a0023d 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -4,16 +4,16 @@ from unittest import mock from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.trial.unittest import TestCase -from scrapy.spiders import Spider -from scrapy.http import Request, Response +from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import _InvalidOutput +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from scrapy.core.spidermw import SpiderMiddlewareManager class SpiderMiddlewareTestCase(TestCase): diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index af17c13a0..e359d9cfc 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,7 +1,7 @@ from unittest import TestCase +from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware -from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index faa8e9091..1d5a887cc 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -2,14 +2,14 @@ import logging from unittest import TestCase from testfixtures import LogCapture -from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer +from twisted.trial.unittest import TestCase as TrialTestCase -from scrapy.utils.test import get_crawler -from scrapy.http import Response, Request -from scrapy.spiders import Spider -from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError +from scrapy.http import Request, Response from scrapy.settings import Settings +from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MockServerSpider diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 380bafe04..ea45b7698 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,10 +1,10 @@ +import warnings from unittest import TestCase from urllib.parse import urlparse -import warnings -from scrapy.http import Response, Request +from scrapy.http import Request, Response +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index dad39b6ee..1bc5ccb9a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,17 +1,11 @@ -from urllib.parse import urlparse -from unittest import TestCase import warnings -from scrapy.http import Response, Request +from unittest import TestCase +from urllib.parse import urlparse -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( - DefaultReferrerPolicy, - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - OriginPolicy, - OriginWhenCrossOriginPolicy, POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, POLICY_ORIGIN, @@ -21,6 +15,11 @@ from scrapy.spidermiddlewares.referer import ( POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, POLICY_UNSAFE_URL, + DefaultReferrerPolicy, + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, RefererMiddleware, ReferrerPolicy, SameOriginPolicy, @@ -28,6 +27,7 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) +from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 22716bdda..9111e4c82 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -2,11 +2,11 @@ from unittest import TestCase from testfixtures import LogCapture +from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware -from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 5c6dccf11..f645f4cce 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,11 +1,12 @@ +import shutil from datetime import datetime from pathlib import Path -import shutil + from twisted.trial import unittest +from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider -from scrapy.exceptions import NotConfigured from scrapy.utils.test import get_crawler diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 0e2441f90..1586f90c5 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -2,16 +2,17 @@ import pickle import sys from queuelib.tests import test_queue as t + +from scrapy.http import Request +from scrapy.item import Field, Item +from scrapy.loader import ItemLoader +from scrapy.selector import Selector from scrapy.squeues import ( _MarshalFifoSerializationDiskQueue, _MarshalLifoSerializationDiskQueue, _PickleFifoSerializationDiskQueue, _PickleLifoSerializationDiskQueue, ) -from scrapy.item import Item, Field -from scrapy.http import Request -from scrapy.loader import ItemLoader -from scrapy.selector import Selector class TestItem(Item): diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 5d9001bb0..b444c32b7 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -4,16 +4,16 @@ import unittest import queuelib -from scrapy.squeues import ( - PickleFifoDiskQueue, - PickleLifoDiskQueue, - MarshalFifoDiskQueue, - MarshalLifoDiskQueue, - FifoMemoryQueue, - LifoMemoryQueue, -) from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.squeues import ( + FifoMemoryQueue, + LifoMemoryQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + PickleFifoDiskQueue, + PickleLifoDiskQueue, +) from scrapy.utils.test import get_crawler """ diff --git a/tests/test_stats.py b/tests/test_stats.py index 2ee04429a..7a8adf638 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,10 +1,10 @@ -from datetime import datetime import unittest +from datetime import datetime from unittest import mock from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector, DummyStatsCollector +from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 9a4eeb04e..d272101b8 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -11,7 +11,7 @@ class ToplevelTestCase(TestCase): self.assertIs(type(scrapy.version_info), tuple) def test_request_shortcut(self): - from scrapy.http import Request, FormRequest + from scrapy.http import FormRequest, Request self.assertIs(scrapy.Request, Request) self.assertIs(scrapy.FormRequest, FormRequest) @@ -27,7 +27,7 @@ class ToplevelTestCase(TestCase): self.assertIs(scrapy.Selector, Selector) def test_item_shortcut(self): - from scrapy.item import Item, Field + from scrapy.item import Field, Item self.assertIs(scrapy.Item, Item) self.assertIs(scrapy.Field, Field) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 3b6428686..c695968d7 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,5 +1,5 @@ -from urllib.parse import urlparse import unittest +from urllib.parse import urlparse class UrlparseTestCase(unittest.TestCase): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index d09335651..746731a2e 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -3,7 +3,7 @@ from unittest import TestCase from pytest import mark -from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor +from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed @mark.usefixtures("reactor_pytest") diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 61a683318..78ed9a7c9 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,7 +1,7 @@ import unittest import warnings -from scrapy.exceptions import UsageError, ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0c86c7e7a..b6a84ee91 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -11,7 +11,6 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect - __doctests__ = ["scrapy.utils.datatypes"] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8d7f33c9a..bb0ebc2a4 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,11 +1,11 @@ import random from pytest import mark -from twisted.trial import unittest -from twisted.internet import reactor, defer +from twisted.internet import defer, reactor from twisted.python.failure import Failure +from twisted.trial import unittest -from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 214deceb2..2d9210410 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,7 @@ import inspect import unittest -from unittest import mock import warnings +from unittest import mock from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index da61f4b0b..d1bf64828 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,6 +1,5 @@ from io import StringIO - -from unittest import mock, TestCase +from unittest import TestCase, mock from scrapy.utils.display import pformat, pprint diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index a34664956..6b2a458bc 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,11 +3,10 @@ from pathlib import Path from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.http import Response +from scrapy.utils.gz import gunzip, gzip_magic_number from tests import tests_datadir - SAMPLEDIR = Path(tests_datadir, "compressed") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 893582a32..ed077440c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,8 +1,8 @@ from pytest import mark from twisted.trial import unittest -from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml -from scrapy.http import XmlResponse, TextResponse, Response +from scrapy.http import Response, TextResponse, XmlResponse +from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 438dd0cdc..eae744df5 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,18 +1,18 @@ -import sys import logging +import sys import unittest from testfixtures import LogCapture from twisted.python.failure import Failure +from scrapy.extensions import telnet from scrapy.utils.log import ( - failure_to_exc_info, - TopLevelFormatter, LogCounterHandler, StreamLogger, + TopLevelFormatter, + failure_to_exc_info, ) from scrapy.utils.test import get_crawler -from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 38a61036c..69793ee75 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,10 +1,10 @@ -import sys import os +import sys import unittest from pathlib import Path from unittest import mock -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, @@ -14,7 +14,6 @@ from scrapy.utils.misc import ( walk_modules, ) - __doctests__ = ["scrapy.utils.misc"] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index b08e5f475..90bd350a5 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,8 +1,8 @@ -import unittest -import os -import tempfile -import shutil import contextlib +import os +import shutil +import tempfile +import unittest import warnings from pathlib import Path diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5caa5b8f2..fbf60ca71 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,20 +5,19 @@ import platform from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback +from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f from scrapy.utils.python import ( - memoizemethod_noargs, + MutableAsyncChain, + MutableChain, binary_is_text, equal_attributes, get_func_args, + memoizemethod_noargs, to_bytes, to_unicode, without_none_values, - MutableChain, - MutableAsyncChain, ) - __doctests__ = ["scrapy.utils.python"] diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d82aa19c6..80e15a60f 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,17 +4,16 @@ from pathlib import Path from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response, TextResponse, HtmlResponse +from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( - response_httprepr, - open_in_browser, - get_meta_refresh, get_base_url, + get_meta_refresh, + open_in_browser, + response_httprepr, response_status_message, ) - __doctests__ = ["scrapy.utils.response"] diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 20aebc2d7..5cdcc7f7c 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,7 +1,7 @@ +import dataclasses import datetime import json import unittest -import dataclasses from decimal import Decimal import attr diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 6fb7b8b82..460ae40c3 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,7 +3,7 @@ import unittest from scrapy import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.utils.spider import iterate_spider_output, iter_spider_classes +from scrapy.utils.spider import iter_spider_classes, iterate_spider_output class MySpider1(Spider): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 45e23f793..c79a1fdce 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,9 +1,9 @@ +import unittest from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -import unittest -from scrapy.utils.template import render_templatefile +from scrapy.utils.template import render_templatefile __doctests__ = ["scrapy.utils.template"] diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9133663d9..65522f0fd 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,13 +4,13 @@ from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( + _is_filesystem_path, add_http_if_no_scheme, guess_scheme, - _is_filesystem_path, strip_url, + url_has_any_extension, url_is_from_any_domain, url_is_from_spider, - url_has_any_extension, ) __doctests__ = ["scrapy.utils.url"] diff --git a/tests/test_webclient.py b/tests/test_webclient.py index aadfe0f40..0042fe8f0 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -6,9 +6,9 @@ import shutil from pathlib import Path import OpenSSL.SSL +from twisted.internet import defer, reactor from twisted.trial import unittest -from twisted.web import server, static, util, resource -from twisted.internet import reactor, defer +from twisted.web import resource, server, static, util try: from twisted.internet.testing import StringTransport @@ -16,12 +16,13 @@ except ImportError: # deprecated in Twisted 19.7.0 # (remove once we bump our requirement past that version) from twisted.test.proto_helpers import StringTransport -from twisted.protocols.policies import WrappingFactory + from twisted.internet.defer import inlineCallbacks +from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory -from scrapy.http import Request, Headers +from scrapy.http import Headers, Request from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode From 7f01e1f0ce106bf810501d53217730b754d35edf Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:43:25 -0600 Subject: [PATCH 11/33] added isort to pre-commit-config --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b93a73453..d67249371 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,7 @@ repos: rev: 22.12.0 hooks: - id: black +- repo: https://github.com/pycqa/isort + rev: 5.11.3 + hooks: + - id: isort From 3054235dc09b1667c5c897f976eafa18845283e1 Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 16:10:57 +0000 Subject: [PATCH 12/33] Don't check robotstxt for local files --- scrapy/downloadermiddlewares/robotstxt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 326c35290..8e9beeeef 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -38,6 +38,8 @@ class RobotsTxtMiddleware: def process_request(self, request, spider): if request.meta.get("dont_obey_robotstxt"): return + if request.url.startswith("data:") or request.url.startswith("file:"): + return d = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d From 33b85a9e2a379b355398e2daf416130bb840167d Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 19:51:20 +0000 Subject: [PATCH 13/33] Test local files aren't processed --- tests/test_downloadermiddleware_robotstxt.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ac08c6006..fd27e637d 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -214,6 +214,19 @@ Disallow: /some/randome/page.html middleware.process_request_2(rp, Request("http://site.local/allowed"), None) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") + def test_robotstxt_local_file(self): + middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) + assert not middleware.process_request( + Request("data:text/plain,Hello World data"), None + ) + assert not middleware.process_request( + Request("file:///tests/sample_data/test_site/nothinghere.html"), None + ) + assert isinstance( + middleware.process_request(Request("http://site.local/allowed"), None), + Deferred, + ) + def assertNotIgnored(self, request, middleware): spider = None # not actually used dfd = maybeDeferred(middleware.process_request, request, spider) From f03b47db05e623189cf7719e647d18e8457494f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 27 Jan 2023 17:35:32 +0100 Subject: [PATCH 14/33] Make NO_CALLBACK a callable --- scrapy/http/request/__init__.py | 52 +++++++++++++++------------------ setup.py | 1 - tests/test_http_request.py | 8 +++-- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 068a4baa3..de13cf264 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,10 +5,8 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from enum import Enum from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union -from typing_extensions import Final from w3lib.url import safe_url_string import scrapy @@ -23,21 +21,22 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -# https://github.com/python/typing/issues/689#issuecomment-561425237 -class NoCallbackType(Enum): - NO_CALLBACK = 0 +def NO_CALLBACK(*args, **kwargs): + """When assigned to the ``callback`` parameter of + :class:`~scrapy.http.Request`, it indicates that the request is not meant + to have a spider callback at all. - -#: When assigned to the ``callback`` parameter of -#: :class:`~scrapy.http.Request`, it indicates that the request is not meant to -#: have a spider callback at all. -#: -#: This value should be used by :ref:`components ` -#: that create and handle their own requests, e.g. through -#: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download -#: middlewares handling such requests can treat them differently from requests -#: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK: Final = NoCallbackType.NO_CALLBACK + This value should be used by :ref:`components ` that + create and handle their own requests, e.g. through + :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download + middlewares handling such requests can treat them differently from requests + intended for the :meth:`~scrapy.Spider.parse` callback. + """ + raise RuntimeError( + "The NO_CALLBACK callback has been called. This is a special callback " + "value intended for requests whose callback is never meant to be " + "called." + ) class Request(object_ref): @@ -67,8 +66,6 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ - callback: Union[None, NoCallbackType, Callable] - errback: Optional[Callable] def __init__( self, @@ -94,8 +91,14 @@ class Request(object_ref): raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority - self._set_xback("callback", callback) - self._set_xback("errback", errback) + if not (callable(callback) or callback is None): + raise TypeError( + f"callback must be a callable, got {type(callback).__name__}" + ) + if not (callable(errback) or errback is None): + raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + self.callback = callback + self.errback = errback self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) @@ -105,15 +108,6 @@ class Request(object_ref): self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) - def _set_xback(self, name: str, value: Optional[Callable]) -> None: - if not ( - callable(value) - or value is None - or (name == "callback" and value is NO_CALLBACK) - ): - raise TypeError(f"{name} must be a callable, got {type(value).__name__}") - setattr(self, name, value) - @property def cb_kwargs(self) -> dict: if self._cb_kwargs is None: diff --git a/setup.py b/setup.py index 9150dac0b..f53334d4e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.3.0", - "typing-extensions>=3.10.0.0", ] extras_require = {} cpython_dependencies = [ diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 233a5f0b2..e800f427f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -314,8 +314,10 @@ class RequestTest(unittest.TestCase): r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, + errback=NO_CALLBACK, ) self.assertIs(r5.callback, NO_CALLBACK) + self.assertIs(r5.errback, NO_CALLBACK) def test_callback_and_errback_type(self): with self.assertRaises(TypeError): @@ -328,8 +330,10 @@ class RequestTest(unittest.TestCase): callback="a_function", errback="a_function", ) - with self.assertRaises(TypeError): - self.request_class("http://example.com", errback=NO_CALLBACK) + + def test_no_callback(self): + with self.assertRaises(RuntimeError): + NO_CALLBACK() def test_from_curl(self): # Note: more curated tests regarding curl conversion are in From e9ee9454f960d7e14b5cca4527bf2b185235bd89 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:59:08 -0600 Subject: [PATCH 15/33] fix .isort.cfg --- .isort.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/.isort.cfg b/.isort.cfg index a29184f0a..f238bf7ea 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,2 @@ [settings] profile = black -multi_line_output = 3 From ef794251f6bd238986c4e90763521a0b3ac02dc6 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 15:00:19 -0600 Subject: [PATCH 16/33] fix scrapy/__init__.py --- scrapy/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 44df3d54b..a757a9290 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -8,11 +8,10 @@ import warnings from twisted import version as _txv +# Declare top-level shortcuts from scrapy.http import FormRequest, Request from scrapy.item import Field, Item from scrapy.selector import Selector - -# Declare top-level shortcuts from scrapy.spiders import Spider __all__ = [ From 4bd48d26138176c83086180172e1cff245d49648 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 15:06:54 -0600 Subject: [PATCH 17/33] added pre-commit action --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 740092dab..6b2f4ef10 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -32,6 +32,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: pre-commit/action@v3.0.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 5dcf8b9015d412919aca99cad0371298f9591b94 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Sun, 29 Jan 2023 00:22:56 -0600 Subject: [PATCH 18/33] fix isort version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d67249371..0534bb142 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,6 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.3 + rev: 5.12.0 hooks: - id: isort From 17354a61b11eb792adbe77fcdfa6b95a5993cc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 10:04:27 +0100 Subject: [PATCH 19/33] Avoid duplicities in CI; remove pylint from pre-commit --- .github/workflows/checks.yml | 16 ++++++---------- .pre-commit-config.yaml | 7 +------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6b2f4ef10..aa79cbc0d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,12 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" - env: - TOXENV: security - - python-version: "3.11" - env: - TOXENV: flake8 - python-version: "3.11" env: TOXENV: pylint @@ -26,13 +20,9 @@ jobs: - python-version: "3.11" env: TOXENV: twinecheck - - python-version: "3.11" - env: - TOXENV: black steps: - uses: actions/checkout@v3 - - uses: pre-commit/action@v3.0.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -44,3 +34,9 @@ jobs: run: | pip install -U tox tox + + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0534bb142..f5fc1285f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,6 @@ repos: rev: 6.0.0 hooks: - id: flake8 -- repo: https://github.com/PyCQA/pylint - rev: v2.15.6 - hooks: - - id: pylint - args: [conftest.py, docs, extras, scrapy, setup.py, tests] - repo: https://github.com/psf/black.git rev: 22.12.0 hooks: @@ -20,4 +15,4 @@ repos: - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - - id: isort + - id: isort From e1699479f6e48ce87dea1e6ed5661fea9ca7b1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 11:54:31 +0100 Subject: [PATCH 20/33] =?UTF-8?q?Fix=20typo:=20download=20middleware=20?= =?UTF-8?q?=E2=86=92=20downloader=20middleware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrey Rakhmatullin --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 75dd2a74f..7afb28db5 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,7 +27,7 @@ def NO_CALLBACK(*args, **kwargs): This value should be used by :ref:`components ` that create and handle their own requests, e.g. through - :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download + :meth:`scrapy.core.engine.ExecutionEngine.download`, so that downloader middlewares handling such requests can treat them differently from requests intended for the :meth:`~scrapy.Spider.parse` callback. """ From 389fd99e79374bad73faf98424c97ac804eb1a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 12:37:34 +0100 Subject: [PATCH 21/33] get_media_requests: support and encourage callback=NO_CALLBACK --- scrapy/pipelines/files.py | 3 ++- scrapy/pipelines/images.py | 3 ++- scrapy/pipelines/media.py | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 01a9c41fe..91fc172b2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -22,6 +22,7 @@ from twisted.internet import defer, threads from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available @@ -517,7 +518,7 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests(self, item, info): urls = ItemAdapter(item).get(self.files_urls_field, []) - return [Request(u) for u in urls] + return [Request(u, callback=NO_CALLBACK) for u in urls] def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 0cfa5665a..9d18144ee 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -13,6 +13,7 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -214,7 +215,7 @@ class ImagesPipeline(FilesPipeline): def get_media_requests(self, item, info): urls = ItemAdapter(item).get(self.images_urls_field, []) - return [Request(u) for u in urls] + return [Request(u, callback=NO_CALLBACK) for u in urls] def item_completed(self, results, item, info): with suppress(KeyError): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 1e921f0b5..679035c5d 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -18,6 +18,10 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) +def _DUMMY_CALLBACK(response): + return response + + class MediaPipeline: LOG_FAILED_RESULTS = True @@ -91,7 +95,10 @@ class MediaPipeline: def _process_request(self, request, info, item): fp = self._fingerprinter.fingerprint(request) - cb = request.callback or (lambda _: _) + if not request.callback or request.callback is NO_CALLBACK: + cb = _DUMMY_CALLBACK + else: + cb = request.callback eb = request.errback request.callback = NO_CALLBACK request.errback = None From 78eaf0671bd50642f68d5b07bec3175298120a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 14:33:11 +0100 Subject: [PATCH 22/33] Remove typing-extensions from tox.ini --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index f2268b0f6..453c28c4c 100644 --- a/tox.ini +++ b/tox.ini @@ -94,7 +94,6 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.3.0 - typing-extensions==3.10.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From c1bbb299d7dc30d03c33cc3eda776ae30ba77d0d Mon Sep 17 00:00:00 2001 From: pankaj1707k <76695979+pankaj1707k@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:30:57 +0530 Subject: [PATCH 23/33] Add and run pre-commit hook 'blacken-docs' Change python code snippets to begin with '.. code-block:: python' to be recognized by the hook for formatting. All snippets under '::' (rst literal blocks) are ignored. --- .pre-commit-config.yaml | 6 + docs/faq.rst | 44 ++- docs/intro/overview.rst | 14 +- docs/intro/tutorial.rst | 120 +++--- docs/topics/asyncio.rst | 6 +- docs/topics/broad-crawls.rst | 40 +- docs/topics/commands.rst | 17 +- docs/topics/components.rst | 8 +- docs/topics/contracts.rst | 38 +- docs/topics/coroutines.rst | 29 +- docs/topics/debug.rst | 35 +- docs/topics/developer-tools.rst | 17 +- docs/topics/downloader-middleware.rst | 50 ++- docs/topics/dynamic-content.rst | 15 +- docs/topics/email.rst | 20 +- docs/topics/exceptions.rst | 8 +- docs/topics/exporters.rst | 35 +- docs/topics/extensions.rst | 24 +- docs/topics/feed-exports.rst | 67 ++-- docs/topics/item-pipeline.rst | 50 ++- docs/topics/items.rst | 30 +- docs/topics/jobs.rst | 6 +- docs/topics/link-extractors.rst | 8 +- docs/topics/loaders.rst | 100 +++-- docs/topics/logging.rst | 109 +++-- docs/topics/media-pipeline.rst | 177 ++++++--- docs/topics/practices.rst | 55 ++- docs/topics/request-response.rst | 176 ++++++--- docs/topics/selectors.rst | 33 +- docs/topics/settings.rst | 202 ++++++---- docs/topics/shell.rst | 5 +- docs/topics/signals.rst | 31 +- docs/topics/spider-middleware.rst | 18 +- docs/topics/spiders.rst | 246 +++++++----- docs/topics/stats.rst | 29 +- sep/sep-001.rst | 549 +++++++++++++------------- sep/sep-002.rst | 44 ++- sep/sep-003.rst | 90 +++-- sep/sep-004.rst | 9 +- sep/sep-005.rst | 33 +- sep/sep-008.rst | 5 +- sep/sep-009.rst | 18 +- sep/sep-012.rst | 6 +- sep/sep-014.rst | 170 ++++---- sep/sep-016.rst | 49 ++- sep/sep-017.rst | 10 +- sep/sep-018.rst | 150 +++---- sep/sep-019.rst | 10 +- sep/sep-020.rst | 50 +-- sep/sep-021.rst | 20 +- 50 files changed, 1821 insertions(+), 1260 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5fc1285f..729682392 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,3 +16,9 @@ repos: rev: 5.12.0 hooks: - id: isort +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==22.12.0 diff --git a/docs/faq.rst b/docs/faq.rst index 8a9ba809b..836420567 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -35,8 +35,9 @@ for parsing HTML responses in Scrapy callbacks. You just have to feed the response's body into a ``BeautifulSoup`` object and extract whatever data you need from it. -Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:: +Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser: +.. code-block:: python from bs4 import BeautifulSoup import scrapy @@ -45,17 +46,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars class ExampleSpider(scrapy.Spider): name = "example" allowed_domains = ["example.com"] - start_urls = ( - 'http://www.example.com/', - ) + start_urls = ("http://www.example.com/",) def parse(self, response): # use lxml to get decent HTML parsing speed - soup = BeautifulSoup(response.text, 'lxml') - yield { - "url": response.url, - "title": soup.h1.string - } + soup = BeautifulSoup(response.text, "lxml") + yield {"url": response.url, "title": soup.h1.string} .. note:: @@ -109,11 +105,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient in most cases. If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings:: +setting the following settings: + +.. code-block:: python DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' - SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' + SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" + SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" While pending requests are below the configured values of :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or @@ -159,11 +157,13 @@ See also other suggestions at `StackOverflow`_. .. note:: Remember to disable :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation:: + your custom implementation: + + .. code-block:: python SPIDER_MIDDLEWARES = { - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, - 'myproject.middlewares.CustomOffsiteMiddleware': 500, + "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 500, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation @@ -235,11 +235,13 @@ What does the response status code 999 means? 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider:: +higher) in your spider: + +.. code-block:: python class MySpider(CrawlSpider): - name = 'myspider' + name = "myspider" download_delay = 2 @@ -351,19 +353,21 @@ How to split an item into multiple items in an item pipeline? input item. :ref:`Create a spider middleware ` instead, and use its :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` -method for this purpose. For example:: +method for this purpose. For example: + +.. code-block:: python from copy import deepcopy from itemadapter import is_item, ItemAdapter - class MultiplyItemsMiddleware: + class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: if is_item(item): adapter = ItemAdapter(item) - for _ in range(adapter['multiply_by']): + for _ in range(adapter["multiply_by"]): yield deepcopy(item) Does Scrapy support IPv6 addresses? diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index cfa6bfa83..495aad091 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -https://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination + +.. code-block:: python import scrapy class QuotesSpider(scrapy.Spider): - name = 'quotes' + name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/tag/humor/', + "https://quotes.toscrape.com/tag/humor/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'author': quote.xpath('span/small/text()').get(), - 'text': quote.css('span.text::text').get(), + "author": quote.xpath("span/small/text()").get(), + "text": quote.css("span.text::text").get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 901a170b4..f5e9b372e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -177,7 +177,9 @@ that generates :class:`scrapy.Request ` objects from URLs, you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider:: +for your spider. + +.. code-block:: python from pathlib import Path @@ -187,13 +189,13 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): page = response.url.split("/")[-2] - filename = f'quotes-{page}.html' + filename = f"quotes-{page}.html" Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each @@ -438,7 +440,9 @@ extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data extracted from the page. To do that, we use the ``yield`` Python keyword -in the callback, as you can see below:: +in the callback, as you can see below: + +.. code-block:: python import scrapy @@ -446,16 +450,16 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } If you run this spider, it will output the extracted data with the log:: @@ -543,7 +547,9 @@ There is also an ``attrib`` property available '/page/2/' Let's see now our spider modified to recursively follow the link to the next -page, extracting data from it:: +page, extracting data from it: + +.. code-block:: python import scrapy @@ -551,18 +557,18 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -594,7 +600,9 @@ A shortcut for creating Requests -------------------------------- As a shortcut for creating Request objects you can use -:meth:`response.follow `:: +:meth:`response.follow ` + +.. code-block:: python import scrapy @@ -602,18 +610,18 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("span small::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -622,57 +630,67 @@ need to call urljoin. Note that ``response.follow`` just returns a Request instance; you still have to yield this Request. You can also pass a selector to ``response.follow`` instead of a string; -this selector should extract necessary attributes:: +this selector should extract necessary attributes: - for href in response.css('ul.pager a::attr(href)'): +.. code-block:: python + + for href in response.css("ul.pager a::attr(href)"): yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href -attribute automatically. So the code can be shortened further:: +attribute automatically. So the code can be shortened further: - for a in response.css('ul.pager a'): +.. code-block:: python + + for a in response.css("ul.pager a"): yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use -:meth:`response.follow_all ` instead:: +:meth:`response.follow_all ` instead: - anchors = response.css('ul.pager a') +.. code-block:: python + + anchors = response.css("ul.pager a") yield from response.follow_all(anchors, callback=self.parse) -or, shortening it further:: +or, shortening it further: - yield from response.follow_all(css='ul.pager a', callback=self.parse) +.. code-block:: python + + yield from response.follow_all(css="ul.pager a", callback=self.parse) More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, -this time for scraping author information:: +this time for scraping author information: + +.. code-block:: python import scrapy class AuthorSpider(scrapy.Spider): - name = 'author' + name = "author" - start_urls = ['https://quotes.toscrape.com/'] + start_urls = ["https://quotes.toscrape.com/"] def parse(self, response): - author_page_links = response.css('.author + a') + author_page_links = response.css(".author + a") yield from response.follow_all(author_page_links, self.parse_author) - pagination_links = response.css('li.next a') + pagination_links = response.css("li.next a") yield from response.follow_all(pagination_links, self.parse) def parse_author(self, response): def extract_with_css(query): - return response.css(query).get(default='').strip() + return response.css(query).get(default="").strip() yield { - 'name': extract_with_css('h3.author-title::text'), - 'birthdate': extract_with_css('.author-born-date::text'), - 'bio': extract_with_css('.author-description::text'), + "name": extract_with_css("h3.author-title::text"), + "birthdate": extract_with_css(".author-born-date::text"), + "bio": extract_with_css(".author-description::text"), } This spider will start from the main page, it will follow all the links to the @@ -720,7 +738,9 @@ spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes -with a specific tag, building the URL based on the argument:: +with a specific tag, building the URL based on the argument: + +.. code-block:: python import scrapy @@ -729,20 +749,20 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'https://quotes.toscrape.com/' - tag = getattr(self, 'tag', None) + url = "https://quotes.toscrape.com/" + tag = getattr(self, "tag", None) if tag is not None: - url = url + 'tag/' + tag + url = url + "tag/" + tag yield scrapy.Request(url, self.parse) def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dbee7146d..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,12 +106,14 @@ Enforcing asyncio as a requirement If you are writing a :ref:`component ` that requires asyncio to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to :ref:`enforce it as a requirement `. For -example:: +example: + +.. code-block:: python from scrapy.utils.reactor import is_asyncio_reactor_installed - class MyComponent: + class MyComponent: def __init__(self): if not is_asyncio_reactor_installed(): raise ValueError( diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0927ac2d2..8be89feb2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply the recommended priority queue use:: +To apply the recommended priority queue use: - SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' +.. code-block:: python + + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" .. _broad-crawls-concurrency: @@ -71,7 +73,9 @@ many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU and memory your crawler will have available. -A good starting point is ``100``:: +A good starting point is ``100``: + +.. code-block:: python CONCURRENT_REQUESTS = 100 @@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of threads handling DNS queries. The DNS queue will be processed faster speeding up establishing of connection and crawling overall. -To increase maximum thread pool size use:: +To increase maximum thread pool size use: + +.. code-block:: python REACTOR_THREADPOOL_MAXSIZE = 20 @@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. -To set the log level use:: +To set the log level use: - LOG_LEVEL = 'INFO' +.. code-block:: python + + LOG_LEVEL = "INFO" Disable cookies =============== @@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve performance by saving some CPU cycles and reducing the memory footprint of your Scrapy crawler. -To disable cookies use:: +To disable cookies use: + +.. code-block:: python COOKIES_ENABLED = False @@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. -To disable retries use:: +To disable retries use: + +.. code-block:: python RETRY_ENABLED = False @@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the case for broad crawls) reduce the download timeout so that stuck requests are discarded quickly and free up capacity to process the next ones. -To reduce the download timeout use:: +To reduce the download timeout use: + +.. code-block:: python DOWNLOAD_TIMEOUT = 15 @@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of request constant per crawl batch, otherwise redirect loops may cause the crawler to dedicate too many resources on any specific domain. -To disable redirects use:: +To disable redirects use: + +.. code-block:: python REDIRECT_ENABLED = False @@ -179,7 +195,9 @@ Pages can indicate it in two ways: "main", "index" website pages. Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `:: +:ref:`AjaxCrawlMiddleware `: + +.. code-block:: python AJAXCRAWL_ENABLED = True diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 362190116..54fd5d663 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -617,7 +617,7 @@ Example: .. code-block:: python - COMMANDS_MODULE = 'mybot.commands' + COMMANDS_MODULE = "mybot.commands" .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -636,10 +636,11 @@ The following example adds ``my_command`` command: from setuptools import setup, find_packages - setup(name='scrapy-mymodule', - entry_points={ - 'scrapy.commands': [ - 'my_command=my_scrapy_module.commands:MyCommand', - ], - }, - ) + setup( + name="scrapy-mymodule", + entry_points={ + "scrapy.commands": [ + "my_command=my_scrapy_module.commands:MyCommand", + ], + }, + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index ca301b827..1ed55f000 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -66,16 +66,18 @@ version mismatch, while :exc:`ValueError` may be better if the issue is the value of a setting. If your requirement is a minimum Scrapy version, you may use -:attr:`scrapy.__version__` to enforce your requirement. For example:: +:attr:`scrapy.__version__` to enforce your requirement. For example: + +.. code-block:: python from pkg_resources import parse_version import scrapy - class MyComponent: + class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('2.7'): + if parse_version(scrapy.__version__) < parse_version("2.7"): raise RuntimeError( f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index c29a3a410..211a0f5f2 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts. This allows you to test each callback of your spider by hardcoding a sample url and check various constraints for how the callback processes the response. Each contract is prefixed with an ``@`` and included in the docstring. See the -following example:: +following example: + +.. code-block:: python def parse(self, response): - """ This function parses a sample response. Some contracts are mingled + """ + This function parses a sample response. Some contracts are mingled with this docstring. @url http://www.amazon.com/s?field-keywords=selfish+gene @@ -64,11 +67,13 @@ Custom Contracts If you find you need more power than the built-in Scrapy contracts you can create and load your own contracts in the project by using the -:setting:`SPIDER_CONTRACTS` setting:: +:setting:`SPIDER_CONTRACTS` setting: + +.. code-block:: python SPIDER_CONTRACTS = { - 'myproject.contracts.ResponseCheck': 10, - 'myproject.contracts.ItemValidate': 10, + "myproject.contracts.ResponseCheck": 10, + "myproject.contracts.ItemValidate": 10, } Each contract must inherit from :class:`~scrapy.contracts.Contract` and can @@ -111,22 +116,26 @@ Raise :class:`~scrapy.exceptions.ContractFail` from .. autoclass:: scrapy.exceptions.ContractFail Here is a demo contract which checks the presence of a custom header in the -response received:: +response received: + +.. code-block:: python from scrapy.contracts import Contract from scrapy.exceptions import ContractFail + class HasHeaderContract(Contract): - """ Demo contract which checks the presence of a custom header - @has_header X-CustomHeader + """ + Demo contract which checks the presence of a custom header + @has_header X-CustomHeader """ - name = 'has_header' + name = "has_header" def pre_process(self, response): for header in self.args: if header not in response.headers: - raise ContractFail('X-CustomHeader not present') + raise ContractFail("X-CustomHeader not present") .. _detecting-contract-check-runs: @@ -135,14 +144,17 @@ Detecting check runs When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is set to the ``true`` string. You can use :data:`os.environ` to perform any change to -your spiders or your settings when ``scrapy check`` is used:: +your spiders or your settings when ``scrapy check`` is used: + +.. code-block:: python import os import scrapy + class ExampleSpider(scrapy.Spider): - name = 'example' + name = "example" def __init__(self): - if os.environ.get('SCRAPY_CHECK'): + if os.environ.get("SCRAPY_CHECK"): pass # Do some scraper adjustments when a check is running diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a1ba4ba5c..a0c005204 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -58,49 +58,58 @@ There are several use cases for coroutines in Scrapy. Code that would return Deferreds when written for previous Scrapy versions, such as downloader middlewares and signal handlers, can be rewritten to be -shorter and cleaner:: +shorter and cleaner: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): adapter = ItemAdapter(item) - adapter['field'] = data + adapter["field"] = data return item def process_item(self, item, spider): adapter = ItemAdapter(item) - dfd = db.get_some_data(adapter['id']) + dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) return dfd -becomes:: +becomes: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): adapter = ItemAdapter(item) - adapter['field'] = await db.get_some_data(adapter['id']) + adapter["field"] = await db.get_some_data(adapter["id"]) return item Coroutines may be used to call asynchronous code. This includes other coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. -This means you can use many useful Python libraries providing such code:: +This means you can use many useful Python libraries providing such code: + +.. code-block:: python class MySpiderDeferred(Spider): # ... async def parse(self, response): - additional_response = await treq.get('https://additional.url') + additional_response = await treq.get("https://additional.url") additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests + class MySpiderAsyncio(Spider): # ... async def parse(self, response): async with aiohttp.ClientSession() as session: - async with session.get('https://additional.url') as additional_response: + async with session.get("https://additional.url") as additional_response: additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests @@ -192,7 +201,9 @@ while maintaining support for older Scrapy versions, you may define :term:`asynchronous generator` version of that method with an alternative name: ``process_spider_output_async``. -For example:: +For example: + +.. code-block:: python class UniversalSpiderMiddleware: def process_spider_output(self, response, result, spider): diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index edbcaf432..b133fcc1e 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -5,21 +5,24 @@ Debugging Spiders ================= This document explains the most common techniques for debugging spiders. -Consider the following Scrapy spider below:: +Consider the following Scrapy spider below: + +.. code-block:: python import scrapy from myproject.items import MyItem + class MySpider(scrapy.Spider): - name = 'myspider' + name = "myspider" start_urls = ( - 'http://example.com/page1', - 'http://example.com/page2', - ) + "http://example.com/page1", + "http://example.com/page2", + ) def parse(self, response): # - # collect `item_urls` + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) @@ -28,7 +31,9 @@ Consider the following Scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) + yield scrapy.Request( + item_details_url, self.parse_details, cb_kwargs={"item": item} + ) def parse_details(self, response, item): # populate more `item` fields @@ -103,10 +108,13 @@ showing the response received and the output. How to debug the situation when .. highlight:: python Fortunately, the :command:`shell` is your bread and butter in this case (see -:ref:`topics-shell-inspect-response`):: +:ref:`topics-shell-inspect-response`): + +.. code-block:: python from scrapy.shell import inspect_response + def parse_details(self, response, item=None): if item: # populate more `item` fields @@ -121,10 +129,13 @@ Open in browser Sometimes you just want to see how a certain response looks in a browser, you can use the ``open_in_browser`` function for that. Here is an example of how -you would use it:: +you would use it: + +.. code-block:: python from scrapy.utils.response import open_in_browser + def parse_details(self, response): if "item name" not in response.body: open_in_browser(response) @@ -138,14 +149,16 @@ Logging Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be -available in all future runs should they be necessary again:: +available in all future runs should they be necessary again: + +.. code-block:: python def parse_details(self, response, item=None): if item: # populate more `item` fields return item else: - self.logger.warning('No item received for %s', response.url) + self.logger.warning("No item received for %s", response.url) For more information, check the :ref:`topics-logging` section. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 9bf97c628..39e7b7d3c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -237,17 +237,19 @@ on the request and open ``Open in new tab`` to get a better overview. :alt: JSON-object returned from the quotes.toscrape API With this response we can now easily parse the JSON-object and -also request each page to get every quote on the site:: +also request each page to get every quote on the site: + +.. code-block:: python import scrapy import json class QuoteSpider(scrapy.Spider): - name = 'quote' - allowed_domains = ['quotes.toscrape.com'] + name = "quote" + allowed_domains = ["quotes.toscrape.com"] page = 1 - start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): data = json.loads(response.text) @@ -275,7 +277,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the :meth:`~scrapy.Request.from_curl()` method to generate an equivalent -request:: +request: + +.. code-block:: python from scrapy import Request @@ -286,7 +290,8 @@ request:: "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW" "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http" - "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") + "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'" + ) Alternatively, if you want to know the arguments needed to recreate that request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 986da0476..e1c481c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the :setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the middleware class paths and their values are the middleware orders. -Here's an example:: +Here's an example: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, + "myproject.middlewares.CustomDownloaderMiddleware": 543, } The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the @@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` -as its value. For example, if you want to disable the user-agent middleware:: +as its value. For example, if you want to disable the user-agent middleware: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, + "myproject.middlewares.CustomDownloaderMiddleware": 543, + "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -226,20 +230,25 @@ There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. -For example:: +For example: + +.. code-block:: python for i, url in enumerate(urls): - yield scrapy.Request(url, meta={'cookiejar': i}, - callback=self.parse_page) + yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page) Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep -passing it along on subsequent requests. For example:: +passing it along on subsequent requests. For example: + +.. code-block:: python def parse_page(self, response): # do some processing - return scrapy.Request("http://www.example.com/otherpage", - meta={'cookiejar': response.meta['cookiejar']}, - callback=self.parse_other_page) + return scrapy.Request( + "http://www.example.com/otherpage", + meta={"cookiejar": response.meta["cookiejar"]}, + callback=self.parse_other_page, + ) .. setting:: COOKIES_ENABLED @@ -339,16 +348,19 @@ HttpAuthMiddleware domain of the first request, which will work for some spiders but not for others. In the future the middleware will produce an error instead. - Example:: + Example: + + .. code-block:: python from scrapy.spiders import CrawlSpider + class SomeIntranetSiteSpider(CrawlSpider): - http_user = 'someuser' - http_pass = 'somepass' - http_auth_domain = 'intranet.example.com' - name = 'intranet.example.com' + http_user = "someuser" + http_pass = "somepass" + http_auth_domain = "intranet.example.com" + name = "intranet.example.com" # .. rest of the spider code omitted ... @@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can specify these in the ``handle_httpstatus_list`` spider attribute. For example, if you want the redirect middleware to ignore 301 and 302 -responses (and pass them through to your spider) you can do this:: +responses (and pass them through to your spider) you can do this: + +.. code-block:: python class MySpider(CrawlSpider): handle_httpstatus_list = [301, 302] diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index ea5d06210..9be0ed058 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -119,16 +119,20 @@ data from it depends on the type of response: ` as usual. - If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `:: + :attr:`response.text `: + + .. code-block:: python data = json.loads(response.text) If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a :class:`~scrapy.Selector` and then - :ref:`use it ` as usual:: + :ref:`use it ` as usual: - selector = Selector(data['html']) + .. code-block:: python + + selector = Selector(data["html"]) - If the response is JavaScript, or HTML with a ``