diff --git a/.bandit.yml b/.bandit.yml index 2aae8a0aa..b7f1817e0 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,21 +1,7 @@ skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B306 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 +- B101 # assert_used, needed for mypy +- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 +- B503 # ssl_with_bad_defaults exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f76bf783d..599cd0cff 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,11 @@ [bumpversion] -current_version = 2.11.0 +current_version = 2.11.2 commit = True tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] + +[bumpversion:file:SECURITY.md] +parse = (?P\d+)\.(?P\d+)\.x +serialize = {major}.{minor}.x diff --git a/.coveragerc b/.coveragerc index ad0ee0f6c..f9ad353d5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,9 @@ include = scrapy/* omit = tests/* disable_warnings = include-ignored + +[report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = + pragma: no cover + if TYPE_CHECKING: diff --git a/.flake8 b/.flake8 index 544d72956..62ccad9cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 119 -ignore = W503, E203 +ignore = E203, E501, E701, E704, W503 exclude = docs/conf.py diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dbcebfa0a..a9fc3dd68 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,7 @@ # .git-blame-ignore-revs # adding black formatter to all the code e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d -# re applying black to the code with default line length +# reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a -# reaplying black again and removing line length on pre-commit black config +# reapplying black again and removing line length on pre-commit black config c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 7a380a7a5..4c0400cde 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..a911d4cfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==24.2.0 diff --git a/MANIFEST.in b/MANIFEST.in index 4920dc0c3..06971e39c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ -include README.rst -include AUTHORS -include INSTALL -include LICENSE -include MANIFEST.in +include CODE_OF_CONDUCT.md +include CONTRIBUTING.md +include INSTALL.md include NEWS +include SECURITY.md include scrapy/VERSION include scrapy/mime.types @@ -12,16 +11,13 @@ include scrapy/py.typed include codecov.yml include conftest.py include pytest.ini -include requirements-*.txt include tox.ini recursive-include scrapy/templates * -recursive-include scrapy license.txt recursive-include docs * prune docs/build recursive-include extras * -recursive-include bin * recursive-include tests * global-exclude __pycache__ *.py[cod] diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..51305d95e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.11.x | :white_check_mark: | +| < 2.11.x | :x: | + +## Reporting a Vulnerability + +Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new. diff --git a/conftest.py b/conftest.py index 68921f119..2ab3dffd4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,6 @@ -import platform -import sys from pathlib import Path import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -85,12 +81,12 @@ def only_not_asyncio(request, reactor_pytest): def requires_uvloop(request): if not request.node.get_closest_marker("requires_uvloop"): return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + try: + import uvloop + + del uvloop + except ImportError: + pytest.skip("uvloop is not installed") def pytest_configure(config): diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..dcd2c9a3a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,6 @@ # serve to show the default. import sys -from datetime import datetime from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -48,7 +47,7 @@ master_doc = "index" # General information about the project. project = "Scrapy" -copyright = f"2008–{datetime.now().year}, Scrapy developers" +copyright = "Scrapy developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -227,7 +226,7 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", ] diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..d728338da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -178,7 +178,7 @@ Scrapy: * We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run black manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..d394406e8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -138,39 +138,37 @@ See previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ -If you have a spider with a long list of -:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider -replacing the default -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware -with a :ref:`custom spider middleware ` that requires -less memory. For example: +If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains` +(e.g. 50,000+), consider replacing the default +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader +middleware with a :ref:`custom downloader middleware +` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in - :attr:`~scrapy.Spider.allowed_domains` into a complex regular - expression. + instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. -See also other suggestions at `StackOverflow`_. +See also `other suggestions at StackOverflow +`__. .. note:: Remember to disable - :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation: + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you + enable your custom implementation: .. code-block:: python - SPIDER_MIDDLEWARES = { - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, - "myproject.middlewares.CustomOffsiteMiddleware": 500, + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 50, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 .. _re: https://docs.python.org/library/re.html -.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -206,12 +204,10 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those messages are thrown by the Offsite Spider Middleware, which is a spider -middleware (enabled by default) whose purpose is to filter out requests to -domains outside the ones covered by the spider. - -For more info see: -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. +Those messages are thrown by +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a +downloader middleware (enabled by default) whose purpose is to filter out +requests to domains outside the ones covered by the spider. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- @@ -297,9 +293,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` uses. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- @@ -405,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a :ref:`topics-stop-response-download` topic for additional information and examples. +.. _faq-blank-request: + +How can I make a blank request? +------------------------------- + +.. code-block:: python + + from scrapy import Request + + + blank_request = Request("data:,") + +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +in-line in web pages as if they were external resources. The "data:" scheme with an empty +content (",") essentially creates a request to a data URL without any specific content. + + Running ``runspider`` I get ``error: No spider found in file: `` -------------------------------------------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index 65d9c5181..758b22d80 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,229 @@ Release notes ============= +.. _release-VERSION: + +Scrapy VERSION (YYYY-MM-DD) +--------------------------- + +Deprecations +~~~~~~~~~~~~ + +- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`6340`) + + +.. _release-2.11.2: + +Scrapy 2.11.2 (2024-05-14) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Redirects to non-HTTP protocols are no longer followed. Please, see the + `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`) + + .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h + +- The ``Authorization`` header is now dropped on redirects to a different + scheme (``http://`` or ``https://``) or port, even if the domain is the + same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more + information. + + .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f + +- When using system proxy settings that are different for ``http://`` and + ``https://``, redirects to a different URL scheme will now also trigger the + corresponding change in proxy settings for the redirected request. Please, + see the `jm3v-qxmh-hxwv security advisory`_ for more information. + (:issue:`767`) + + .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv + +- :attr:`Spider.allowed_domains ` is now + enforced for all requests, and not only requests from spider callbacks. + (:issue:`1042`, :issue:`2241`, :issue:`6358`) + +- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML + entities. (:issue:`6265`) + +- defusedxml_ is now used to make + :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure. + (:issue:`6250`, :issue:`6251`) + + .. _defusedxml: https://github.com/tiran/defusedxml + +Bug fixes +~~~~~~~~~ + +- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in + favor of brotli_. (:issue:`6261`) + + .. _brotli: https://github.com/google/brotli + + .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli + instead if you can. + +- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This + prevents + :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from + following redirects that would not be followed by web browsers with + JavaScript enabled. (:issue:`6342`, :issue:`6347`) + +- During :ref:`feed export `, do not close the + underlying file from :ref:`built-in post-processing plugins + `. + (:issue:`5932`, :issue:`6178`, :issue:`6239`) + +- :class:`LinkExtractor ` + now properly applies the ``unique`` and ``canonicalize`` parameters. + (:issue:`3273`, :issue:`6221`) + +- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty + string. (:issue:`6121`, :issue:`6124`) + +- Fix :attr:`Spider.logger ` not logging custom extra + information. (:issue:`6323`, :issue:`6324`) + +- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing + the UTF-8-compatible (e.g. ASCII) parts of the document. + (:issue:`6292`, :issue:`6298`) + +- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an + exception if ``default`` is ``None``. + (:issue:`6308`, :issue:`6310`) + +- :class:`~scrapy.selector.Selector` now uses + :func:`scrapy.utils.response.get_base_url` to determine the base URL of a + given :class:`~scrapy.http.Response`. (:issue:`6265`) + +- The :meth:`media_to_download` method of :ref:`media pipelines + ` now logs exceptions before stripping them. + (:issue:`5067`, :issue:`5068`) + +- When passing a callback to the :command:`parse` command, build the callback + callable with the right signature. + (:issue:`6182`) + +Documentation +~~~~~~~~~~~~~ + +- Add a FAQ entry about :ref:`creating blank requests `. + (:issue:`6203`, :issue:`6208`) + +- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``. + (:issue:`6328`, :issue:`6334`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Make builds reproducible. (:issue:`5019`, :issue:`6322`) + +- Packaging and test fixes. + (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`) + + +.. _release-2.11.1: + +Scrapy 2.11.1 (2024-02-14) +-------------------------- + +Highlights: + +- Security bug fixes. + +- Support for Twisted >= 23.8.0. + +- Documentation improvements. + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Addressed `ReDoS vulnerabilities`_: + + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees when using libxml2 2.7+. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + + .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the + deprecated ``scrapy.downloadermiddlewares.decompression`` module has been + removed. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`, + :issue:`6064`, :issue:`6142`) + +Bug fixes +~~~~~~~~~ + +- The OS signal handling code was refactored to no longer use private Twisted + functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`) + +Documentation +~~~~~~~~~~~~~ + +- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization + changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) + +- Extended documentation for :attr:`Request.meta `. + (:issue:`5565`) + +- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, + :issue:`6077`) + +- Added a link to Zyte's export guides to the :ref:`feed exports + ` documentation. (:issue:`6183`) + +- Added a missing note about backward-incompatible changes in + :class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes. + (:issue:`6060`, :issue:`6081`) + +- Added a missing note about removing the deprecated + ``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes. + (:issue:`6056`, :issue:`6061`) + +- Other documentation improvements. (:issue:`6128`, :issue:`6144`, + :issue:`6163`, :issue:`6190`, :issue:`6192`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added Python 3.12 to the CI configuration, re-enabled tests that were + disabled when the pre-release support was added. (:issue:`5985`, + :issue:`6083`, :issue:`6098`) + +- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -62,6 +285,9 @@ Deprecation removals 1.0.0, use :attr:`CrawlerRunner.spider_loader ` instead. (:issue:`6010`) +- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in + Scrapy 2.6.0, has now been removed. (:issue:`6111`) + Deprecations ~~~~~~~~~~~~ @@ -1157,6 +1383,9 @@ Deprecations Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` first to set the :class:`~scrapy.Spider` object. +- :func:`scrapy.utils.response.response_httprepr` is now deprecated. + (:issue:`4972`) + New features ~~~~~~~~~~~~ @@ -2871,6 +3100,38 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (2024-02-14) +------------------------- + +**Security bug fixes:** + +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using libxml2 2.7+. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the + ``scrapy.downloadermiddlewares.decompression`` module is discouraged and + will trigger a warning. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: diff --git a/docs/requirements.txt b/docs/requirements.txt index 9f9aef711..5f683d34c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==5.0.2 -sphinx-hoverxref==1.1.1 -sphinx-notfound-page==0.8 -sphinx-rtd-theme==1.0.0 +sphinx==6.2.1 +sphinx-hoverxref==1.3.0 +sphinx-notfound-page==1.0.0 +sphinx-rtd-theme==2.0.0 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..d2fc41003 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -150,8 +150,7 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..8a13b8976 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,6 +47,18 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. +Disabling throttling on a downloader slot +========================================= + +It is possible to disable AutoThrottle for a specific download slot at run time +by setting its ``throttle`` attribute to ``False``, e.g. using +:setting:`DOWNLOAD_SLOTS`. + +Note, however, that AutoThrottle still determines the starting delay of every +slot by setting the ``download_delay`` attribute on the running spider. You +might want to set a custom value for the ``delay`` attribute of the slot, e.g. +using :setting:`DOWNLOAD_SLOTS`. + Throttling algorithm ==================== @@ -131,7 +143,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 0643df6a6..b704e54ed 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -24,7 +24,8 @@ You should see an output like this:: 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.corestats.CoreStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: - ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', @@ -37,7 +38,6 @@ You should see an output like this:: 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 8be89feb2..750aae554 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -116,7 +116,7 @@ Reduce log level When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when preforming large broad crawls in +should not use ``DEBUG`` log level when performing large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..c31f7fe43 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -763,6 +763,44 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +OffsiteMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.offsite + :synopsis: Offsite Middleware + +.. class:: OffsiteMiddleware + + .. versionadded:: 2.11.2 + + Filters out Requests for URLs outside the domains covered by the spider. + + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. + + When your spider returns a request for a domain not belonging to those + covered by the spider, this middleware will log a debug message similar to + this one:: + + DEBUG: Filtered offsite request to 'offsite.example': + + To avoid filling the log with too much noise, it will only print one of + these messages for each new domain filtered. So, for example, if another + request for ``offsite.example`` is filtered, no log message will be + printed. But if a request for ``other.example`` is filtered, a message + will be printed (but only for the first request filtered). + + If the spider doesn't define an + :attr:`~scrapy.Spider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute + set, the offsite middleware will allow the request even if its domain is not + listed in allowed domains. + RedirectMiddleware ------------------ @@ -882,7 +920,15 @@ Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``["script", "noscript"]`` to ``[]``. + +.. versionchanged:: 2.11.2 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``[]`` to ``["noscript"]``. + +.. versionchanged:: VERSION + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``[]`` to ``['noscript']``. .. setting:: METAREFRESH_MAXDELAY diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 700775e4b..922b765db 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which allows you to generate feeds with the scraped items, using multiple serialization formats and storage backends. +This page provides detailed documentation for all feed export features. If you +are looking for a step-by-step guide, check out `Zyte’s export guides`_. + +.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data + .. _topics-feed-format: Serialization formats @@ -385,7 +390,13 @@ Each plugin is a class that must implement the following methods: .. method:: close(self) - Close the target file object. + Clean up the plugin. + + For example, you might want to close a file wrapper that you might have + used to compress data written into the file received in the ``__init__`` + method. + + .. warning:: Do not close the file from the ``__init__`` method. To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 3c38ac2dc..97ed7a900 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -399,12 +399,7 @@ In code that receives an item, such as methods of :ref:`item pipelines `, it is a good practice to use the :class:`~itemadapter.ItemAdapter` class and the :func:`~itemadapter.is_item` function to write code that works for -any :ref:`supported item type `: - -.. autoclass:: itemadapter.ItemAdapter - -.. autofunction:: itemadapter.is_item - +any supported item type. Other classes related to items ============================== diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index da0587aa4..c96dd0f99 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.files import FilesPipeline class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. @@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.images import ImagesPipeline class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b1b8c9e9c..cd359b147 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ + plugin `__ and additional + features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8edf710bc..3c2843bc1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -94,13 +94,14 @@ Request objects .. code-block:: python request_with_cookies = Request( - url="http://www.example.com", + url="https://www.example.com", cookies=[ { "name": "currency", "value": "USD", "domain": "example.com", "path": "/currency", + "secure": True, }, ], ) @@ -469,60 +470,6 @@ import path. .. autoclass:: scrapy.utils.request.RequestFingerprinter - -.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION - -REQUEST_FINGERPRINTER_IMPLEMENTATION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.7 - -Default: ``'2.6'`` - -Determines which request fingerprinting algorithm is used by the default -request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). - -Possible values are: - -- ``'2.6'`` (default) - - This implementation uses the same request fingerprinting algorithm as - Scrapy 2.6 and earlier versions. - - Even though this is the default value for backward compatibility reasons, - it is a deprecated value. - -- ``'2.7'`` - - This implementation was introduced in Scrapy 2.7 to fix an issue of the - previous implementation. - - New projects should use this value. The :command:`startproject` command - sets this value in the generated ``settings.py`` file. - -If you are using the default value (``'2.6'``) for this setting, and you are -using Scrapy components where changing the request fingerprinting algorithm -would cause undesired results, you need to carefully decide when to change the -value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the 2.6 request -fingerprinting algorithm and does not log this warning ( -:ref:`2.6-request-fingerprinter` includes an example implementation of such a -class). - -Scenarios where changing the request fingerprinting algorithm may cause -undesired results include, for example, using the HTTP cache middleware (see -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidate the current -cache, requiring you to redownload all requests again. - -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in -your settings to switch already to the request fingerprinting implementation -that will be the only request fingerprinting implementation available in a -future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'2.6'``). - - -.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter @@ -731,6 +678,7 @@ Those are: * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` +* :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) @@ -1357,3 +1305,13 @@ XmlResponse objects line. See :attr:`TextResponse.encoding`. .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241 + +JsonResponse objects +-------------------- + +.. class:: JsonResponse(url[, ...]) + + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` + header. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 4a64d530b..e32fc2b70 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1032,10 +1032,8 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions. - -.. autofunction:: parsel.xpathfuncs.set_xpathfunc - +Parsel also simplifies adding your own XPath extensions with +:func:`~parsel.xpathfuncs.set_xpathfunc`. .. _topics-selectors-ref: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06f8481ba..904bd7ecc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -674,6 +674,7 @@ Default: .. code-block:: python { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -835,7 +836,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -844,7 +845,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -856,6 +862,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + There is no global setting for ``throttle``, whose default value is + ``None``. + .. setting:: DOWNLOAD_TIMEOUT @@ -873,40 +882,42 @@ The amount of time (in secs) that the downloader will wait before timing out. Request.meta key. .. setting:: DOWNLOAD_MAXSIZE +.. reqmeta:: download_maxsize DOWNLOAD_MAXSIZE ---------------- -Default: ``1073741824`` (1024MB) +Default: ``1073741824`` (1 GiB) -The maximum response size (in bytes) that downloader will download. +The maximum response body size (in bytes) allowed. Bigger responses are +aborted and ignored. -If you want to disable it set to 0. +This applies both before and after compression. If decompressing a response +body would exceed this limit, decompression is aborted and the response is +ignored. -.. reqmeta:: download_maxsize +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_maxsize` - spider attribute and per-request using :reqmeta:`download_maxsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_maxsize` spider +attribute and per request using the :reqmeta:`download_maxsize` Request.meta +key. .. setting:: DOWNLOAD_WARNSIZE +.. reqmeta:: download_warnsize DOWNLOAD_WARNSIZE ----------------- -Default: ``33554432`` (32MB) +Default: ``33554432`` (32 MiB) -The response size (in bytes) that downloader will start to warn. +If the size of a response exceeds this value, before or after compression, a +warning will be logged about it. -If you want to disable it set to 0. +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_warnsize` - spider attribute and per-request using :reqmeta:`download_warnsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_warnsize` spider +attribute and per request using the :reqmeta:`download_warnsize` Request.meta +key. .. setting:: DOWNLOAD_FAIL_ON_DATALOSS @@ -1603,7 +1614,6 @@ Default: { "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..13e636055 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -343,11 +343,18 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.Request`, to be - downloaded later. + Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be + downloaded later, before the request reaches the :ref:`scheduler + `. + + Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it + reaches the scheduler. This signal does not support returning deferreds from its handlers. + .. versionadded:: 2.11.2 + Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. + :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3f16efea5..8ddf17a14 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -51,8 +51,8 @@ value. For example, if you want to disable the off-site middleware: .. code-block:: python SPIDER_MIDDLEWARES = { - "myproject.middlewares.CustomSpiderMiddleware": 543, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "scrapy.spidermiddlewares.referer.RefererMiddleware": None, + "myproject.middlewares.CustomRefererSpiderMiddleware": 700, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -313,42 +313,6 @@ Default: ``False`` Pass all responses, regardless of its status code. -OffsiteMiddleware ------------------ - -.. module:: scrapy.spidermiddlewares.offsite - :synopsis: Offsite Spider Middleware - -.. class:: OffsiteMiddleware - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'www.othersite.com': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``www.othersite.com`` is filtered, no log message will be - printed. But if a request for ``someothersite.com`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. - RefererMiddleware ----------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 30677fe74..8a0102a51 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -75,7 +75,8 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if - :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 832829b75..0e4a8fa6c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -172,8 +172,8 @@ TELNETCONSOLE_PORT Default: ``[6023, 6073]`` -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. +The port range to use for the telnet console. If set to ``None``, a dynamically +assigned port is used. .. setting:: TELNETCONSOLE_HOST diff --git a/pylintrc b/pylintrc index c8654b8d3..c60e4e16a 100644 --- a/pylintrc +++ b/pylintrc @@ -4,21 +4,14 @@ jobs=1 # >1 hides results [MESSAGES CONTROL] disable=abstract-method, - anomalous-backslash-in-string, arguments-differ, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-mcs-classmethod-argument, bare-except, broad-except, broad-exception-raised, c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, consider-using-with, cyclic-import, dangerous-default-value, @@ -32,7 +25,6 @@ disable=abstract-method, implicit-str-concat, import-error, import-outside-toplevel, - import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -44,7 +36,6 @@ disable=abstract-method, logging-fstring-interpolation, logging-not-lazy, lost-exception, - method-hidden, missing-docstring, no-else-raise, no-else-return, @@ -52,7 +43,7 @@ disable=abstract-method, no-method-argument, no-name-in-module, no-self-argument, - no-value-for-parameter, + no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, pointless-exception-statement, pointless-statement, @@ -77,23 +68,15 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, - unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, - unsubscriptable-object, unused-argument, unused-import, unused-private-member, unused-variable, unused-wildcard-import, - use-dict-literal, used-before-assignment, - useless-object-inheritance, # Required for Python 2 support useless-return, - useless-super-delegation, wildcard-import, wrong-import-position diff --git a/scrapy/VERSION b/scrapy/VERSION index 46b81d815..9e5bb77a3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.0 +2.11.2 diff --git a/scrapy/addons.py b/scrapy/addons.py index 9060d4f3f..65d7a0310 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -32,9 +32,7 @@ class AddonManager: for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) - addon = create_instance( - addoncls, settings=settings, crawler=self.crawler - ) + addon = build_from_crawler(addoncls, self.crawler) addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6580ba9ce..da0e51386 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,21 +1,33 @@ +from __future__ import annotations + import argparse import cProfile import inspect import os import sys from importlib.metadata import entry_points +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec + + _P = ParamSpec("_P") + class ScrapyArgumentParser(argparse.ArgumentParser): - def _parse_optional(self, arg_string): + def _parse_optional( + self, arg_string: str + ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -23,7 +35,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name): +def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -37,8 +49,8 @@ def _iter_command_classes(module_name): yield obj -def _get_commands_from_module(module, inproject): - d = {} +def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: + d: Dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -46,8 +58,10 @@ def _get_commands_from_module(module, inproject): return d -def _get_commands_from_entry_points(inproject, group="scrapy.commands"): - cmds = {} +def _get_commands_from_entry_points( + inproject: bool, group: str = "scrapy.commands" +) -> Dict[str, ScrapyCommand]: + cmds: Dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -61,7 +75,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"): return cmds -def _get_commands_dict(settings, inproject): +def _get_commands_dict( + settings: BaseSettings, inproject: bool +) -> Dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -70,16 +86,17 @@ def _get_commands_dict(settings, inproject): return cmds -def _pop_command_name(argv): +def _pop_command_name(argv: List[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): del argv[i] return arg i += 1 + return None -def _print_header(settings, inproject): +def _print_header(settings: BaseSettings, inproject: bool) -> None: version = scrapy.__version__ if inproject: print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") @@ -88,7 +105,7 @@ def _print_header(settings, inproject): print(f"Scrapy {version} - no active project\n") -def _print_commands(settings, inproject): +def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) print("Usage:") print(" scrapy [options] [args]\n") @@ -103,13 +120,20 @@ def _print_commands(settings, inproject): print('Use "scrapy -h" to see more info about a command') -def _print_unknown_command(settings, cmdname, inproject): +def _print_unknown_command( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: _print_header(settings, inproject) print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') -def _run_print_help(parser, func, *a, **kw): +def _run_print_help( + parser: argparse.ArgumentParser, + func: Callable[_P, None], + *a: _P.args, + **kw: _P.kwargs, +) -> None: try: func(*a, **kw) except UsageError as e: @@ -120,7 +144,9 @@ def _run_print_help(parser, func, *a, **kw): sys.exit(2) -def execute(argv=None, settings=None): +def execute( + argv: Optional[List[str]] = None, settings: Optional[Settings] = None +) -> None: if argv is None: argv = sys.argv @@ -162,14 +188,16 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) -def _run_command(cmd, args, opts): +def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) -def _run_command_profiled(cmd, args, opts): +def _run_command_profiled( + cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace +) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..9fe803d3c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,62 +1,64 @@ """ Base class for Scrapy commands """ + import argparse +import builtins import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False + requires_project: bool = False crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} - exitcode = 0 + exitcode: int = 0 def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline - def set_crawler(self, crawler): + def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") - self._crawler = crawler + self._crawler: Crawler = crawler - def syntax(self): + def syntax(self) -> str: """ Command syntax (preferably one-line). Do not include command name. """ return "" - def short_desc(self): + def short_desc(self) -> str: """ A short description of the command """ return "" - def long_desc(self): + def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() - def help(self): + def help(self) -> str: """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: """ Populate option parse with options available for this command """ @@ -91,7 +93,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args, opts): + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -128,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand): Common class used to share functionality between the crawl, parse and runspider commands """ - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-a", dest="spargs", @@ -161,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: @@ -182,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Help Formatter for scrapy command line help messages. """ - def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + def __init__( + self, + prog: str, + indent_increment: int = 2, + max_help_position: int = 24, + width: Optional[int] = None, + ): super().__init__( prog, indent_increment=indent_increment, @@ -190,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): width=width, ) - def _join_parts(self, part_strings): - parts = self.format_part_strings(part_strings) + def _join_parts(self, part_strings: Iterable[str]) -> str: + # scrapy.commands.list shadows builtins.list + parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings): + def format_part_strings(self, part_strings: List[str]) -> List[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..0c4ebcd23 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,10 +1,14 @@ -import subprocess +import argparse +import subprocess # nosec import sys import time +from typing import Any, Iterable, List from urllib.parse import urlencode import scrapy +from scrapy import Request from scrapy.commands import ScrapyCommand +from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor @@ -15,24 +19,28 @@ class Command(ScrapyCommand): "CLOSESPIDER_TIMEOUT": 10, } - def short_desc(self): + def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: with _BenchServer(): + assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() class _BenchServer: - def __enter__(self): + def __enter__(self) -> None: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) # nosec + assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: self.proc.kill() self.proc.wait() time.sleep(0.2) @@ -47,11 +55,12 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self): + def start_requests(self) -> Iterable[Request]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response): + def parse(self, response: Response) -> Any: + assert isinstance(Response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index de54ca4d3..22c8abf7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,5 +1,7 @@ +import argparse import time from collections import defaultdict +from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): - def printSummary(self, start, stop): + def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - writeln = self.stream.writeln + # _WritelnDecorator isn't implemented in typeshed yet + writeln = self.stream.writeln # type: ignore[attr-defined] run = self.testsRun plural = "s" if run != 1 else "" @@ -42,14 +45,14 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Check spider contracts" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -66,7 +69,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) @@ -76,6 +79,7 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader with set_environ(SCRAPY_CHECK="true"): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2f0f1c7b9..6e023af81 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,3 +1,8 @@ +import argparse +from typing import List, cast + +from twisted.python.failure import Failure + from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): requires_project = True - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a spider" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: @@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand): ) spname = args[0] + assert self.crawler_process crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) if getattr(crawl_defer, "result", None) is not None and issubclass( - crawl_defer.result.type, Exception + cast(Failure, crawl_defer.result).type, Exception ): self.exitcode = 1 else: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..04012bee8 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,7 @@ +import argparse import os import sys +from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -9,32 +11,34 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "" - def short_desc(self): + def short_desc(self) -> str: return "Edit spider" - def long_desc(self): + def long_desc(self) -> str: return ( "Edit a spider using the editor defined in the EDITOR environment" " variable or else the EDITOR setting" ) - def _err(self, msg): + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() editor = self.settings["EDITOR"] + assert self.crawler_process try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ + assert sfile sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cdb7ad4ae..1acf2d26f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,13 +1,13 @@ import sys -from argparse import Namespace -from typing import List, Type +from argparse import ArgumentParser, Namespace +from typing import Dict, List, Type from w3lib.url import is_url from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request @@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): requires_project = False - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Fetch a URL using the Scrapy downloader" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and print its content" " to stdout. You may want to use --nolog to disable logging" ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -44,20 +44,21 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers, prefix): + def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: Namespace) -> None: if opts.headers: + assert response.request self._print_headers(response.request.headers, b">") print(">") self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) - def _print_bytes(self, bytes_): + def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args: List[str], opts: Namespace) -> None: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..2649fb23d 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,9 +1,10 @@ +import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, cast +from typing import List, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -def sanitize_module_name(module_name): +def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start with one @@ -23,7 +24,7 @@ def sanitize_module_name(module_name): return module_name -def extract_domain(url): +def extract_domain(url: str) -> str: """Extract domain name from URL string""" o = urlparse(url) if o.scheme == "" and o.netloc == "": @@ -31,7 +32,7 @@ def extract_domain(url): return o.netloc -def verify_url_scheme(url): +def verify_url_scheme(url: str) -> str: """Check url for scheme and insert https if none found.""" parsed = urlparse(url) if parsed.scheme == "" and parsed.netloc == "": @@ -43,14 +44,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Generate new spider using pre-defined templates" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -86,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return @@ -113,9 +114,16 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # nosec - def _genspider(self, module, name, url, template_name, template_file): + def _genspider( + self, + module: str, + name: str, + url: str, + template_name: str, + template_file: Union[str, os.PathLike], + ) -> None: """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) domain = extract_domain(url) @@ -130,6 +138,7 @@ class Command(ScrapyCommand): } if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) + assert spiders_module.__file__ spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None @@ -152,7 +161,7 @@ class Command(ScrapyCommand): print('Use "scrapy genspider --list" to see all available templates.') return None - def _list_templates(self): + def _list_templates(self) -> None: print("Available templates:") for file in sorted(Path(self.templates_dir).iterdir()): if file.suffix == ".tmpl": diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2f5032360..dcc51a694 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + from scrapy.commands import ScrapyCommand @@ -5,9 +8,10 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def short_desc(self): + def short_desc(self) -> str: return "List available spiders" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ac937e464..2453c0d39 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,15 +1,32 @@ +import argparse +import functools import inspect import json import logging -from typing import Dict +from types import CoroutineType +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + TypeVar, + Union, + overload, +) from itemadapter import ItemAdapter, is_item -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -19,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, list] = {} - requests: Dict[int, list] = {} + items: Dict[int, List[Any]] = {} + requests: Dict[int, List[Request]] = {} first_response = None - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" - def add_options(self, parser): - BaseRunSpiderCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--spider", dest="spider", @@ -105,7 +124,7 @@ class Command(BaseRunSpiderCommand): ) @property - def max_level(self): + def max_level(self) -> int: max_items, max_requests = 0, 0 if self.items: max_items = max(self.items) @@ -113,13 +132,21 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) - def handle_exception(self, _failure): + def handle_exception(self, _failure: Failure) -> None: logger.error( "An error is caught while iterating the async iterable", exc_info=failure_to_exc_info(_failure), ) - def iterate_spider_output(self, result): + @overload + def iterate_spider_output( + self, result: Union[AsyncGenerator, CoroutineType] + ) -> Deferred: ... + + @overload + def iterate_spider_output(self, result: _T) -> Iterable: ... + + def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -132,15 +159,15 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl, new_items): + def add_items(self, lvl: int, new_items: List[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl, new_reqs): + def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl=None, colour=True): + def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -149,7 +176,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl=None, colour=True): + def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -161,7 +188,7 @@ class Command(BaseRunSpiderCommand): print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) - def print_results(self, opts): + def print_results(self, opts: argparse.Namespace) -> None: colour = not opts.nocolour if opts.verbose: @@ -178,7 +205,14 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): + def _get_items_and_requests( + self, + spider_output: Iterable[Any], + opts: argparse.Namespace, + depth: int, + spider: Spider, + callback: Callable, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -187,14 +221,21 @@ class Command(BaseRunSpiderCommand): requests.append(x) return items, requests, opts, depth, spider, callback - def run_callback(self, response, callback, cb_kwargs=None): + def run_callback( + self, + response: Response, + callback: Callable, + cb_kwargs: Optional[Dict[str, Any]] = None, + ) -> Deferred: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d - def get_callback_from_rules(self, spider, response): + def get_callback_from_rules( + self, spider: Spider, response: Response + ) -> Union[Callable, str, None]: if getattr(spider, "rules", None): - for rule in spider.rules: + for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: @@ -203,8 +244,10 @@ class Command(BaseRunSpiderCommand): "please specify a callback to use for parsing", {"spider": spider.name}, ) + return None - def set_spidercls(self, url, opts): + def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: try: @@ -218,13 +261,14 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider): + def _start_requests(spider: Spider) -> Iterable[Request]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: self.spidercls.start_requests = _start_requests - def start_parsing(self, url, opts): + def start_parsing(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() @@ -232,7 +276,12 @@ class Command(BaseRunSpiderCommand): if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) - def scraped_data(self, args): + def scraped_data( + self, + args: Tuple[ + List[Any], List[Request], argparse.Namespace, int, Spider, Callable + ], + ) -> List[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -251,42 +300,53 @@ class Command(BaseRunSpiderCommand): return scraped_data - def prepare_request(self, spider, request, opts): - def callback(response, **cb_kwargs): + def _get_callback( + self, + *, + spider: Spider, + opts: argparse.Namespace, + response: Optional[Response] = None, + ) -> Callable: + cb: Union[str, Callable, None] = None + if response: + cb = response.meta["_callback"] + if not cb: + if opts.callback: + cb = opts.callback + elif response and opts.rules and self.first_response == response: + cb = self.get_callback_from_rules(spider, response) + if not cb: + raise ValueError( + f"Cannot find a rule that matches {response.url!r} in spider: " + f"{spider.name}" + ) + else: + cb = "parse" + + if not callable(cb): + assert cb is not None + cb_method = getattr(spider, cb, None) + if callable(cb_method): + cb = cb_method + else: + raise ValueError( + f"Cannot find callback {cb!r} in spider: {spider.name}" + ) + assert callable(cb) + return cb + + def prepare_request( + self, spider: Spider, request: Request, opts: argparse.Namespace + ) -> Request: + def callback(response: Response, **cb_kwargs: Any) -> Deferred: # memorize first request if not self.first_response: self.first_response = response - # determine real callback - cb = response.meta["_callback"] - if not cb: - if opts.callback: - cb = opts.callback - elif opts.rules and self.first_response == response: - cb = self.get_callback_from_rules(spider, response) - - if not cb: - logger.error( - "Cannot find a rule that matches %(url)r in spider: %(spider)s", - {"url": response.url, "spider": spider.name}, - ) - return - else: - cb = "parse" - - if not callable(cb): - cb_method = getattr(spider, cb, None) - if callable(cb_method): - cb = cb_method - else: - logger.error( - "Cannot find callback %(callback)r in spider: %(spider)s", - {"callback": cb, "spider": spider.name}, - ) - return + cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests - depth = response.meta["_depth"] + depth: int = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -303,16 +363,19 @@ class Command(BaseRunSpiderCommand): request.meta["_depth"] = 1 request.meta["_callback"] = request.callback + if not request.callback and not opts.rules: + cb = self._get_callback(spider=spider, opts=opts) + functools.update_wrapper(callback, cb) request.callback = callback return request - def process_options(self, args, opts): - BaseRunSpiderCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_request_meta(self, opts): + def process_request_meta(self, opts: argparse.Namespace) -> None: if opts.meta: try: opts.meta = json.loads(opts.meta) @@ -323,7 +386,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def process_request_cb_kwargs(self, opts): + def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) @@ -334,7 +397,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 58ed89a81..77850e7b5 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,9 +1,10 @@ +import argparse import sys from importlib import import_module from os import PathLike from pathlib import Path from types import ModuleType -from typing import Union +from typing import List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand): requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a self-contained spider (without creating a project)" - def long_desc(self): + def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) @@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand): raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() + assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 318187204..dbda73b44 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,6 @@ +import argparse import json +from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -8,14 +10,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options]" - def short_desc(self): + def short_desc(self) -> str: return "Get settings values" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--get", dest="get", metavar="SETTING", help="print raw setting value" ) @@ -44,7 +46,8 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process settings = self.crawler_process.settings if opts.get: s = settings.get(opts.get) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..668c95a7b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,9 +3,10 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import Namespace + +from argparse import ArgumentParser, Namespace from threading import Thread -from typing import List, Type +from typing import Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -23,20 +24,20 @@ class Command(ScrapyCommand): "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } - def syntax(self): + def syntax(self) -> str: return "[url|file]" - def short_desc(self): + def short_desc(self) -> str: return "Interactive scraping console" - def long_desc(self): + def long_desc(self) -> str: return ( "Interactive console for scraping the given url or file. " "Use ./file.html syntax or full path for local file." ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-c", dest="code", @@ -51,7 +52,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars): + def update_vars(self, vars: Dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ @@ -87,7 +88,8 @@ class Command(ScrapyCommand): shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) shell.start(url=url, redirect=not opts.no_redirect) - def _start_crawler_thread(self): + def _start_crawler_thread(self) -> None: + assert self.crawler_process t = Thread( target=self.crawler_process.start, kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index fde609c6f..58c1aa28f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,4 @@ +import argparse import os import re import string @@ -5,13 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import List, Tuple, Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER = ( +TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path): +def _make_writable(path: Union[str, os.PathLike]) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) @@ -31,14 +33,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return " [project_dir]" - def short_desc(self): + def short_desc(self) -> str: return "Create new project" - def _is_valid_name(self, project_name): - def _module_exists(module_name): + def _is_valid_name(self, project_name: str) -> bool: + def _module_exists(module_name: str) -> bool: spec = find_spec(module_name) return spec is not None and spec.loader is not None @@ -53,7 +55,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src: Path, dst: Path): + def _copytree(self, src: Path, dst: Path) -> None: """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -84,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -105,7 +107,9 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - move(project_dir / "module", project_dir / project_name) + # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case + # See https://bugs.python.org/issue32689 + move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 47582866b..f057e8544 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions @@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[-v]" - def short_desc(self): + def short_desc(self) -> str: return "Print Scrapy version" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--verbose", "-v", @@ -22,7 +25,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index ebdfa10a8..21679e3aa 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,21 +1,28 @@ import argparse +import logging from scrapy.commands import fetch +from scrapy.http import Response, TextResponse from scrapy.utils.response import open_in_browser +logger = logging.getLogger(__name__) + class Command(fetch.Command): - def short_desc(self): + def short_desc(self) -> str: return "Open URL in browser, as seen by Scrapy" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and show its contents in a browser" ) - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--headers", help=argparse.SUPPRESS) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: argparse.Namespace) -> None: + if not isinstance(response, TextResponse): + logger.error("Cannot view a non-text response.") + return open_in_browser(response) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..0ab3bdb77 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,8 +1,9 @@ import random +import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -10,6 +11,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response from scrapy.resolver import dnscache from scrapy.settings import BaseSettings @@ -24,10 +26,18 @@ if TYPE_CHECKING: class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + def __init__( + self, + concurrency: int, + delay: float, + randomize_delay: bool, + *, + throttle: Optional[bool] = None, + ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay + self.throttle = throttle self.active: Set[Request] = set() self.queue: Deque[Tuple[Request, Deferred]] = deque() @@ -40,7 +50,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec return self.delay def close(self) -> None: @@ -52,13 +62,15 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" + f"randomize_delay={self.randomize_delay!r}, " + f"throttle={self.throttle!r})" ) def __str__(self) -> str: return ( f"" @@ -115,7 +127,7 @@ class Downloader: return len(self.active) >= self.total_concurrency def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: - key = self._get_slot_key(request, spider) + key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( @@ -127,12 +139,13 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) + throttle = slot_settings.get("throttle", None) + new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) self.slots[key] = new_slot return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Spider) -> str: + def get_slot_key(self, request: Request) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) @@ -142,6 +155,14 @@ class Downloader: return key + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + warnings.warn( + "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.get_slot_key(request) + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 909cc273f..0e77cd2fe 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import TYPE_CHECKING, Any, List, Optional @@ -19,12 +21,16 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -62,7 +68,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): method: int = SSL.SSLv23_METHOD, *args: Any, **kwargs: Any, - ): + ) -> Self: tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) @@ -97,7 +103,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - ctx = self.getCertificateOptions().getContext() + ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx @@ -160,23 +166,23 @@ class AcceptableProtocolsContextFactory: return options -def load_context_factory_from_settings(settings, crawler): +def load_context_factory_from_settings( + settings: BaseSettings, crawler: Crawler +) -> IPolicyForHTTPS: ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory try: - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, method=ssl_method, ) except TypeError: # use context factory defaults - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, ) msg = ( f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 6a211aafa..ade51ca63 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: @@ -21,9 +21,9 @@ logger = logging.getLogger(__name__) class DownloadHandlers: def __init__(self, crawler: "Crawler"): self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing + self._schemes: Dict[str, Union[str, Callable]] = ( + {} + ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( @@ -55,10 +55,9 @@ class DownloadHandlers: dhcls = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None - dh = create_instance( - objcls=dhcls, - settings=self._crawler.settings, - crawler=self._crawler, + dh = build_from_crawler( + dhcls, + self._crawler, ) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 4824167da..17dd7483b 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,6 +2,8 @@ from pathlib import Path from w3lib.url import file_uri_to_path +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -10,7 +12,7 @@ class FileDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: filepath = file_uri_to_path(request.url) body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 4081545ce..77dcf3c38 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -28,34 +28,46 @@ In case of status 200 request, response.headers will come with two keys: 'Size' - with size of the downloaded data """ +from __future__ import annotations + import re from io import BytesIO +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote +from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient +from twisted.python.failure import Failure +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class ReceivedDataProtocol(Protocol): - def __init__(self, filename=None): - self.__filename = filename - self.body = open(filename, "wb") if filename else BytesIO() - self.size = 0 + def __init__(self, filename: Optional[str] = None): + self.__filename: Optional[str] = filename + self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + self.size: int = 0 - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.body.write(data) self.size += len(data) @property - def filename(self): + def filename(self) -> Optional[str]: return self.__filename - def close(self): + def close(self) -> None: self.body.close() if self.filename else self.body.seek(0) @@ -65,21 +77,21 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING = { + CODE_MAPPING: Dict[str, int] = { "550": 404, "default": 503, } - def __init__(self, settings): + def __init__(self, settings: BaseSettings): self.default_user = settings["FTP_USER"] self.default_password = settings["FTP_PASSWORD"] self.passive_mode = settings["FTP_PASSIVE_MODE"] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -91,28 +103,29 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client, request, filepath): + def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol).addCallbacks( - callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,), - ) + d = client.retrieveFile(filepath, protocol) + d.addCallback(self._build_response, request, protocol) + d.addErrback(self._failed, request) + return d - def _build_response(self, result, request, protocol): + def _build_response( + self, result: Any, request: Request, protocol: ReceivedDataProtocol + ) -> Response: self.result = result protocol.close() headers = {"local filename": protocol.filename or "", "size": protocol.size} body = to_bytes(protocol.filename or protocol.body.read()) respcls = responsetypes.from_args(url=request.url, body=body) - return respcls(url=request.url, status=200, body=body, headers=headers) + # hints for Headers-related types may need to be fixed to not use AnyStr + return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] - def _failed(self, result, request): + def _failed(self, result: Failure, request: Request) -> Response: message = result.getErrorMessage() if result.type == CommandFailed: m = _CODE_RE.search(message) @@ -122,4 +135,5 @@ class FTPDownloadHandler: return Response( url=request.url, status=httpcode, body=to_bytes(message) ) + assert result.type raise result.type(result.value) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 6c1dac4a5..da9559525 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,39 +1,57 @@ """Download handlers for http and https schemes """ -from scrapy.utils.misc import create_instance, load_object + +from __future__ import annotations + +from typing import TYPE_CHECKING, Type + +from twisted.internet.defer import Deferred + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.settings import BaseSettings +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory + from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + class HTTP10DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): - self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) - self.ClientContextFactory = load_object( + def __init__(self, settings: BaseSettings, crawler: Crawler): + self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + settings["DOWNLOADER_HTTPCLIENTFACTORY"] + ) + self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) - self._settings = settings - self._crawler = crawler + self._settings: BaseSettings = settings + self._crawler: Crawler = crawler @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) return factory.deferred - def _connect(self, factory): + def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port if factory.scheme == b"https": - client_context_factory = create_instance( - objcls=self.ClientContextFactory, - settings=self._settings, - crawler=self._crawler, + client_context_factory = build_from_crawler( + self.ClientContextFactory, + self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c3704de3d..5e84be6ba 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,65 +1,77 @@ """Download handlers for http and https schemes""" +from __future__ import annotations + import ipaddress import logging import re from contextlib import suppress from io import BytesIO from time import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urldefrag, urlunparse -from twisted.internet import defer, protocol, ssl +from twisted.internet import ssl +from twisted.internet.base import ReactorBase +from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from twisted.internet.interfaces import IConsumer +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import ( - URI, - Agent, - HTTPConnectionPool, - ResponseDone, - ResponseFailed, -) +from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import Response as TxResponse +from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS from zope.interface import implementer -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse +from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings: BaseSettings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor - self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) self._pool._factory.noisy = False - self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") - self._disconnect_timeout = 1 + self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( + settings, crawler + ) + self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") + self._disconnect_timeout: int = 1 @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -71,10 +83,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self): + def close(self) -> Deferred: from twisted.internet import reactor - d = self._pool.closeCachedConnections() + d: Deferred = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -85,7 +97,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result): + def cancel_delayed_call(result: Any) -> Any: if delayed_call.active(): delayed_call.cancel() return result @@ -115,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__( self, - reactor, - host, - port, - proxyConf, - contextFactory, - timeout=30, - bindAddress=None, + reactor: ReactorBase, + host: str, + port: int, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: IPolicyForHTTPS, + timeout: float = 30, + bindAddress: Optional[Tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred = defer.Deferred() - self._tunneledHost = host - self._tunneledPort = port - self._contextFactory = contextFactory - self._connectBuffer = bytearray() + self._tunnelReadyDeferred: Deferred = Deferred() + self._tunneledHost: str = host + self._tunneledPort: int = port + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectBuffer: bytearray = bytearray() - def requestTunnel(self, protocol): + def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" + assert protocol.transport tunnelReq = tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived - protocol.dataReceived = self.processProxyResponse + protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign] self._protocol = protocol return protocol - def processProxyResponse(self, rcvd_bytes): + def processProxyResponse(self, data: bytes) -> None: """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ - self._connectBuffer += rcvd_bytes + assert self._protocol.transport + self._connectBuffer += data # make sure that enough (all) bytes are consumed # and that we've got all HTTP headers (ending with a blank line) # from the proxy so that we don't send those bytes to the TLS layer @@ -155,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # see https://github.com/scrapy/scrapy/issues/2491 if b"\r\n\r\n" not in self._connectBuffer: return - self._protocol.dataReceived = self._protocolDataReceived + self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( + sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] self._tunneledHost, self._tunneledPort ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + extra: Any if respm: extra = { "status": int(respm.group("status")), "reason": respm.group("reason").strip(), } else: - extra = rcvd_bytes[: self._truncatedLength] + extra = data[: self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError( "Could not open CONNECT tunnel with proxy " @@ -179,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ) ) - def connectFailed(self, reason): + def connectFailed(self, reason: Failure) -> None: """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory): + def connect(self, protocolFactory: Factory) -> Deferred: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -191,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def tunnel_request_data( + host: str, port: int, proxy_auth_header: Optional[bytes] = None +) -> bytes: r""" Return binary content of a CONNECT request. @@ -222,18 +239,19 @@ class TunnelingAgent(Agent): def __init__( self, - reactor, - proxyConf, - contextFactory=None, - connectTimeout=None, - bindAddress=None, - pool=None, + *, + reactor: ReactorBase, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: IPolicyForHTTPS, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf = proxyConf - self._contextFactory = contextFactory + self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri): + def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: return TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, @@ -245,8 +263,15 @@ class TunnelingAgent(Agent): ) def _requestWithEndpoint( - self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath - ): + self, + key: Any, + endpoint: TCP4ClientEndpoint, + method: bytes, + parsedURI: bytes, + headers: Optional[TxHeaders], + bodyProducer: Optional[IBodyProducer], + requestPath: bytes, + ) -> Deferred: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -264,7 +289,12 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__( - self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + self, + reactor: ReactorBase, + proxyURI: bytes, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): super().__init__( reactor=reactor, @@ -272,9 +302,15 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI: URI = URI.fromBytes(proxyURI) - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[TxHeaders] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> Deferred: """ Issue a new request via the configured proxy. """ @@ -298,26 +334,27 @@ class ScrapyAgent: def __init__( self, - contextFactory=None, - connectTimeout=10, - bindAddress=None, - pool=None, - maxsize=0, - warnsize=0, - fail_on_dataloss=True, - crawler=None, + *, + contextFactory: IPolicyForHTTPS, + connectTimeout: float = 10, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, + maxsize: int = 0, + warnsize: int = 0, + fail_on_dataloss: bool = True, + crawler: Crawler, ): - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._txresponse = None - self._crawler = crawler + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectTimeout: float = connectTimeout + self._bindAddress: Optional[bytes] = bindAddress + self._pool: Optional[HTTPConnectionPool] = pool + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._txresponse: Optional[TxResponse] = None + self._crawler: Crawler = crawler - def _get_agent(self, request, timeout): + def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress @@ -325,10 +362,10 @@ class ScrapyAgent: if proxy: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - proxyHost = to_unicode(proxyHost) + proxyHost_str = to_unicode(proxyHost) if scheme == b"https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost, proxyPort, proxyAuth) + proxyConf = (proxyHost_str, proxyPort, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -338,7 +375,9 @@ class ScrapyAgent: pool=self._pool, ) proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) + proxyURI = urlunparse( + (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") + ) return self._ProxyAgent( reactor=reactor, proxyURI=to_bytes(proxyURI, encoding="ascii"), @@ -355,7 +394,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request): + def download_request(self, request: Request) -> Deferred: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -372,7 +411,7 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d = agent.request( + d: Deferred = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency @@ -385,7 +424,9 @@ class ScrapyAgent: d.addBoth(self._cb_timeout, request, url, timeout) return d - def _cb_timeout(self, result, request, url, timeout): + def _cb_timeout( + self, result: Any, request: Request, url: str, timeout: float + ) -> Any: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -396,19 +437,21 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result, request, start_time): + def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: request.meta["download_latency"] = time() - start_time return result @staticmethod - def _headers_from_twisted_response(response): + def _headers_from_twisted_response(response: TxResponse) -> Headers: headers = Headers() if response.length != UNKNOWN_LENGTH: headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers - def _cb_bodyready(self, txresponse, request): + def _cb_bodyready( + self, txresponse: TxResponse, request: Request + ) -> Union[Dict[str, Any], Deferred]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -464,7 +507,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) txresponse._transport.loseConnection() - raise defer.CancelledError(warning_msg % warning_args) + raise CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning( @@ -473,11 +516,11 @@ class ScrapyAgent: {"size": expected_size, "warnsize": warnsize, "request": request}, ) - def _cancel(_): + def _cancel(_: Any) -> None: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d = defer.Deferred(_cancel) + d: Deferred = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -495,7 +538,9 @@ class ScrapyAgent: return d - def _cb_bodydone(self, result, request, url): + def _cb_bodydone( + self, result: Dict[str, Any], request: Request, url: str + ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -515,53 +560,57 @@ class ScrapyAgent: ) if result.get("failure"): result["failure"].value.response = response - return result["failure"] + return cast(Failure, result["failure"]) return response @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): + def __init__(self, body: bytes): self.body = body self.length = len(body) - def startProducing(self, consumer): + def startProducing(self, consumer: IConsumer) -> Deferred: consumer.write(self.body) - return defer.succeed(None) + return succeed(None) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass -class _ResponseReader(protocol.Protocol): +class _ResponseReader(Protocol): def __init__( self, - finished, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - crawler, + finished: Deferred, + txresponse: TxResponse, + request: Request, + maxsize: int, + warnsize: int, + fail_on_dataloss: bool, + crawler: Crawler, ): - self._finished = finished - self._txresponse = txresponse - self._request = request - self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._fail_on_dataloss_warned = False - self._reached_warnsize = False - self._bytes_received = 0 - self._certificate = None - self._ip_address = None - self._crawler = crawler + self._finished: Deferred = finished + self._txresponse: TxResponse = txresponse + self._request: Request = request + self._bodybuf: BytesIO = BytesIO() + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._fail_on_dataloss_warned: bool = False + self._reached_warnsize: bool = False + self._bytes_received: int = 0 + self._certificate: Optional[ssl.Certificate] = None + self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( + None + ) + self._crawler: Crawler = crawler - def _finish_response(self, flags=None, failure=None): + def _finish_response( + self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + ) -> None: self._finished.callback( { "txresponse": self._txresponse, @@ -573,7 +622,8 @@ class _ResponseReader(protocol.Protocol): } ) - def connectionMade(self): + def connectionMade(self) -> None: + assert self.transport if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( @@ -585,11 +635,12 @@ class _ResponseReader(protocol.Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes): + def dataReceived(self, bodyBytes: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return + assert self.transport self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) @@ -636,7 +687,7 @@ class _ResponseReader(protocol.Protocol): {"warnsize": self._warnsize, "request": self._request}, ) - def connectionLost(self, reason): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self._finished.called: return diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b2579362c..16fc1e3ae 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,11 +1,14 @@ +from __future__ import annotations + from time import time -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI +from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -16,13 +19,13 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes -H2DownloadHandlerOrSubclass = TypeVar( - "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" -) +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class H2DownloadHandler: - def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): + def __init__(self, settings: Settings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor @@ -31,9 +34,7 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler( - cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler - ) -> H2DownloadHandlerOrSubclass: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -54,7 +55,7 @@ class ScrapyH2Agent: def __init__( self, - context_factory, + context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, bind_address: Optional[bytes] = None, diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 81d8e8115..9a0811a50 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,21 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Type + +from twisted.internet.defer import Deferred + +from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class S3DownloadHandler: def __init__( self, - settings, + settings: BaseSettings, *, - crawler=None, - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, - **kw, + crawler: Crawler, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + **kw: Any, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -43,6 +56,8 @@ class S3DownloadHandler: if kw: raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: + assert aws_access_key_id is not None + assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] self._signer = SignerCls( botocore.credentials.Credentials( @@ -50,18 +65,17 @@ class S3DownloadHandler: ) ) - _http_handler = create_instance( - objcls=httpdownloadhandler, - settings=settings, - crawler=crawler, + _http_handler = build_from_crawler( + httpdownloadhandler, + crawler, ) self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs): + def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname @@ -78,6 +92,7 @@ class S3DownloadHandler: headers=request.headers.to_unicode_dict(), data=request.body, ) + assert self._signer self._signer.add_auth(awsrequest) request = request.replace(url=url, headers=awsrequest.headers.items()) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..52ebe4e22 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,6 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd1f56f8c..4eca03800 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ + import logging from time import time from typing import ( @@ -27,14 +28,15 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider +from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -291,9 +293,19 @@ class ExecutionEngine: self.slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request, spider: Spider) -> None: - self.signals.send_catch_log( - signals.request_scheduled, request=request, spider=spider + request_scheduled_result = self.signals.send_catch_log( + signals.request_scheduled, + request=request, + spider=spider, + dont_log=IgnoreRequest, ) + for handler, result in request_scheduled_result: + if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): + logger.debug( + f"Signal handler {global_object_name(handler)} dropped " + f"request {request} before it reached the scheduler." + ) + return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=spider @@ -346,7 +358,7 @@ class ExecutionEngine: assert self.spider is not None dwld = self.downloader.fetch(request, self.spider) - dwld.addCallbacks(_on_success) + dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld @@ -358,16 +370,15 @@ class ExecutionEngine: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._next_request) - scheduler = create_instance( - self.scheduler_cls, settings=None, crawler=self.crawler - ) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests( start_requests, spider ) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open"): - yield scheduler.open(spider) + if d := scheduler.open(spider): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 39d5921f4..4132fc385 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,7 +2,6 @@ import logging from enum import Enum from io import BytesIO from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -from urllib.parse import urlparse from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -111,17 +111,17 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream @@ -185,7 +185,7 @@ class Stream: def check_request_url(self) -> bool: # Make sure that we are sending the request to the correct URL - url = urlparse(self._request.url) + url = urlparse_cached(self._request) return ( url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") @@ -194,7 +194,7 @@ class Stream: ) def _get_request_headers(self) -> List[Tuple[str, str]]: - url = urlparse(self._request.url) + url = urlparse_cached(self._request) path = url.path if url.query: diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 17c95f1ea..f30a5d9c9 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,19 +4,23 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, cast from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + # requires queuelib >= 1.6.2 + from queuelib.queue import BaseQueue + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -121,9 +125,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") - - class Scheduler(BaseScheduler): """ Default Scrapy scheduler. This implementation also handles duplication @@ -179,30 +180,30 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass=None, - mqclass=None, + dqclass: Optional[Type[BaseQueue]] = None, + mqclass: Optional[Type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass=None, + pqclass: Optional[Type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass = pqclass - self.dqclass = dqclass - self.mqclass = mqclass + self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[Type[BaseQueue]] = dqclass + self.mqclass: Optional[Type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) return cls( - dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + dupefilter=build_from_crawler(dupefilter_cls, crawler), jobdir=job_dir(crawler.settings), dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), @@ -221,9 +222,9 @@ class Scheduler(BaseScheduler): (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory (3) return the result of the dupefilter's ``open`` method """ - self.spider = spider - self.mqs = self._mq() - self.dqs = self._dq() if self.dqdir else None + self.spider: Spider = spider + self.mqs: ScrapyPriorityQueue = self._mq() + self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() def close(self, reason: str) -> Optional[Deferred]: @@ -320,24 +321,26 @@ class Scheduler(BaseScheduler): return self.dqs.pop() return None - def _mq(self): + def _mq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with in-memory storage""" - return create_instance( + assert self.crawler + assert self.pqclass + return build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.mqclass, key="", ) - def _dq(self): + def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" + assert self.crawler assert self.dqdir + assert self.pqclass state = self._read_dqs_state(self.dqdir) - q = create_instance( + q = build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.dqclass, key=self.dqdir, startprios=state, diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..566e6628b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,5 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging @@ -7,7 +8,6 @@ from collections import deque from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, Deque, Generator, @@ -17,6 +17,7 @@ from typing import ( Tuple, Type, Union, + cast, ) from itemadapter import is_item @@ -183,7 +184,9 @@ class Scraper: result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback(self.handle_spider_output, request, result, spider) + dfd.addCallback( + self.handle_spider_output, request, cast(Response, result), spider + ) return dfd def _scrape2( @@ -255,12 +258,12 @@ class Scraper: self, result: Union[Iterable, AsyncIterable], request: Request, - response: Union[Response, Failure], + response: Response, spider: Spider, ) -> Deferred: if not result: return defer_succeed(None) - it: Union[Generator, AsyncGenerator] + it: Union[Iterable, AsyncIterable] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dcf1a6dbc..2cef2e1dd 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice @@ -103,8 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Union[Generator, AsyncGenerator]: def process_sync(iterable: Iterable) -> Generator: try: - for r in iterable: - yield r + yield from iterable except Exception as ex: exception_result = self._process_spider_exception( response, spider, Failure(ex), exception_processor_index @@ -303,10 +303,8 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallbacks( - callback=deferred_f_from_coro_f(process_callback_output), - errback=process_spider_exception, - ) + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + dfd.addErrback(process_spider_exception) return dfd def process_start_requests( diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d3a11208..ccfe78891 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -39,7 +39,7 @@ from scrapy.utils.log import ( log_reactor_info, log_scrapy_info, ) -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( install_reactor, @@ -109,10 +109,9 @@ class Crawler: lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter = create_instance( + self.request_fingerprinter = build_from_crawler( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, + self, ) reactor_class: str = self.settings["TWISTED_REACTOR"] @@ -179,6 +178,48 @@ class Crawler: assert self.engine yield maybeDeferred(self.engine.stop) + @staticmethod + def _get_component(component_class, components): + for component in components: + if isinstance(component, component_class): + return component + return None + + def get_addon(self, cls): + return self._get_component(cls, self.addons.addons) + + def get_downloader_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_downloader_middleware() can only be called after " + "the crawl engine has been created." + ) + return self._get_component(cls, self.engine.downloader.middleware.middlewares) + + def get_extension(self, cls): + if not self.extensions: + raise RuntimeError( + "Crawler.get_extension() can only be called after the " + "extension manager has been created." + ) + return self._get_component(cls, self.extensions.middlewares) + + def get_item_pipeline(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_item_pipeline() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.itemproc.middlewares) + + def get_spider_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_spider_middleware() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.spidermw.middlewares) + class CrawlerRunner: """ @@ -404,7 +445,9 @@ class CrawlerProcess(CrawlerRunner): d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 85781efd6..6ada3b474 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) +_UNSET = object() def _is_public_domain(domain: str) -> bool: @@ -133,6 +134,7 @@ class CookiesMiddleware: Decode from bytes if necessary. """ decoded = {} + flags = set() for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): @@ -152,10 +154,16 @@ class CookiesMiddleware: cookie, ) decoded[key] = cookie[key].decode("latin1", errors="replace") - + for flag in ("secure",): + value = cookie.get(flag, _UNSET) + if value is _UNSET or not value: + continue + flags.add(flag) cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain cookie_str += f"; {key.capitalize()}={value}" + for flag in flags: # secure + cookie_str += f"; {flag.capitalize()}" return cookie_str def _get_request_cookies( @@ -168,9 +176,11 @@ class CookiesMiddleware: return [] cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies + for cookie in cookies: + cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) response = Response(request.url, headers={"Set-Cookie": formatted}) return jar.make_cookies(response, request) diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..58fd415b9 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,6 +3,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..fd7c03a38 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,6 +3,7 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Union diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56a58a750..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,50 +1,96 @@ from __future__ import annotations -import io -import zlib +import warnings +from itertools import chain +from logging import getLogger from typing import TYPE_CHECKING, List, Optional, Union -from scrapy import Request, Spider +from scrapy import Request, Spider, signals from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.statscollectors import StatsCollector +from scrapy.utils._compression import ( + _DecompressionMaxSizeExceeded, + _inflate, + _unbrotli, + _unzstd, +) +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self +logger = getLogger(__name__) + ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli - - ACCEPTED_ENCODINGS.append(b"br") + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"br") try: - import zstandard - - ACCEPTED_ENCODINGS.append(b"zstd") + import zstandard # noqa: F401 except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"zstd") class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats: Optional[StatsCollector] = None): - self.stats = stats + def __init__( + self, + stats: Optional[StatsCollector] = None, + *, + crawler: Optional[Crawler] = None, + ): + if not crawler: + self.stats = stats + self._max_size = 1073741824 + self._warn_size = 33554432 + return + self.stats = crawler.stats + self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - return cls(stats=crawler.stats) + try: + return cls(crawler=crawler) + except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'crawler' parameter or " + "reimplement their 'from_crawler' method.", + ScrapyDeprecationWarning, + ) + mw = cls() + mw.stats = crawler.stats + mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + crawler.signals.connect(mw.open_spider, signals.spider_opened) + return mw + + def open_spider(self, spider): + if hasattr(spider, "download_maxsize"): + self._max_size = spider.download_maxsize + if hasattr(spider, "download_warnsize"): + self._warn_size = spider.download_warnsize def process_request( self, request: Request, spider: Spider @@ -60,8 +106,26 @@ class HttpCompressionMiddleware: if isinstance(response, Response): content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: - encoding = content_encoding.pop() - decoded_body = self._decode(response.body, encoding.lower()) + max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) + try: + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size + ) + except _DecompressionMaxSizeExceeded: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)} B compressed) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during " + f"decompression." + ) + if len(response.body) < warn_size <= len(decoded_body): + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." + ) + response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -74,7 +138,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs = {"cls": respcls, "body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable @@ -85,25 +149,35 @@ class HttpCompressionMiddleware: return response - def _decode(self, body: bytes, encoding: bytes) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body) + def _handle_encoding(self, body, content_encoding, max_size): + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + def _split_encodings(self, content_encoding): + to_keep = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in ACCEPTED_ENCODINGS: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + + def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: + if encoding in {b"gzip", b"x-gzip"}: + return gunzip(body, max_size=max_size) if encoding == b"deflate": - try: - body = zlib.decompress(body) - except zlib.error: - # ugly hack to work with raw deflate content that may - # be sent by microsoft servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - body = zlib.decompress(body, -15) + return _inflate(body, max_size=max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - body = brotli.decompress(body) + return _unbrotli(body, max_size=max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - # Using its streaming API since its simple API could handle only cases - # where there is content size data embedded in the frame - reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - body = reader.read() + return _unzstd(body, max_size=max_size) return body diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 335896ac1..5b56ad449 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -60,26 +60,33 @@ class HttpProxyMiddleware: def process_request( self, request: Request, spider: Spider ) -> Union[Request, Response, None]: - creds, proxy_url = None, None + creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: creds, proxy_url = self._get_proxy(request.meta["proxy"], "") elif self.proxies: parsed = urlparse_cached(request) - scheme = parsed.scheme + _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - scheme not in ("http", "https") + _scheme not in ("http", "https") or (parsed.hostname and not proxy_bypass(parsed.hostname)) - ) and scheme in self.proxies: + ) and _scheme in self.proxies: + scheme = _scheme creds, proxy_url = self.proxies[scheme] - self._set_proxy_and_creds(request, proxy_url, creds) + self._set_proxy_and_creds(request, proxy_url, creds, scheme) return None def _set_proxy_and_creds( - self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + self, + request: Request, + proxy_url: Optional[str], + creds: Optional[bytes], + scheme: Optional[str], ) -> None: + if scheme: + request.meta["_scheme_proxy"] = True if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py new file mode 100644 index 000000000..1e5026925 --- /dev/null +++ b/scrapy/downloadermiddlewares/offsite.py @@ -0,0 +1,77 @@ +import logging +import re +import warnings + +from scrapy import signals +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.httpobj import urlparse_cached + +logger = logging.getLogger(__name__) + + +class OffsiteMiddleware: + @classmethod + def from_crawler(cls, crawler): + o = cls(crawler.stats) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) + return o + + def __init__(self, stats): + self.stats = stats + self.domains_seen = set() + + def spider_opened(self, spider): + self.host_regex = self.get_host_regex(spider) + + def request_scheduled(self, request, spider): + self.process_request(request, spider) + + def process_request(self, request, spider): + if request.dont_filter or self.should_follow(request, spider): + return None + domain = urlparse_cached(request).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {"domain": domain, "request": request}, + extra={"spider": spider}, + ) + self.stats.inc_value("offsite/domains", spider=spider) + self.stats.inc_value("offsite/filtered", spider=spider) + raise IgnoreRequest + + def should_follow(self, request, spider): + regex = self.host_regex + # hostname can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or "" + return bool(regex.search(host)) + + def get_host_regex(self, spider): + """Override this method to implement a different offsite policy""" + allowed_domains = getattr(spider, "allowed_domains", None) + if not allowed_domains: + return re.compile("") # allow all by default + url_pattern = re.compile(r"^https?://.*$") + port_pattern = re.compile(r":\d+$") + domains = [] + for domain in allowed_domains: + if domain is None: + continue + if url_pattern.match(domain): + message = ( + "allowed_domains accepts only domains, not URLs. " + f"Ignoring URL entry {domain} in allowed_domains." + ) + warnings.warn(message) + elif port_pattern.search(domain): + message = ( + "allowed_domains accepts only domains without ports. " + f"Ignoring entry {domain} in allowed_domains." + ) + warnings.warn(message) + else: + domains.append(re.escape(domain)) + regex = rf'^(.*\.)?({"|".join(domains)})$' + return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 814b1a561..aa08827c4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, List, Union, cast -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -29,11 +29,49 @@ def _build_redirect_request( **kwargs, cookies=None, ) - if "Cookie" in redirect_request.headers: - source_request_netloc = urlparse_cached(source_request).netloc - redirect_request_netloc = urlparse_cached(redirect_request).netloc - if source_request_netloc != redirect_request_netloc: + if "_scheme_proxy" in redirect_request.meta: + source_request_scheme = urlparse_cached(source_request).scheme + redirect_request_scheme = urlparse_cached(redirect_request).scheme + if source_request_scheme != redirect_request_scheme: + redirect_request.meta.pop("_scheme_proxy") + redirect_request.meta.pop("proxy", None) + redirect_request.meta.pop("_auth_proxy", None) + redirect_request.headers.pop(b"Proxy-Authorization", None) + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: + default_ports = {"http": 80, "https": 443} + + parsed_source_request = urlparse_cached(source_request) + source_scheme, source_host, source_port = ( + parsed_source_request.scheme, + parsed_source_request.hostname, + parsed_source_request.port + or default_ports.get(parsed_source_request.scheme), + ) + + parsed_redirect_request = urlparse_cached(redirect_request) + redirect_scheme, redirect_host, redirect_port = ( + parsed_redirect_request.scheme, + parsed_redirect_request.hostname, + parsed_redirect_request.port + or default_ports.get(parsed_redirect_request.scheme), + ) + + if has_cookie_header and ( + redirect_scheme not in {source_scheme, "https"} + or source_host != redirect_host + ): del redirect_request.headers["Cookie"] + + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header and ( + source_scheme != redirect_scheme + or source_host != redirect_host + or source_port != redirect_port + ): + del redirect_request.headers["Authorization"] + return redirect_request @@ -119,13 +157,15 @@ class RedirectMiddleware(BaseRedirectMiddleware): assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): - request_scheme = urlparse(request.url).scheme + request_scheme = urlparse_cached(request).scheme location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) + redirected = _build_redirect_request(request, url=redirected_url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response if response.status in (301, 307, 308) or request.method == "HEAD": - redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) @@ -147,12 +187,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): request.meta.get("dont_redirect", False) or request.method == "HEAD" or not isinstance(response, HtmlResponse) + or urlparse_cached(request).scheme not in {"http", "https"} ): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and cast(float, interval) < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) + if not url: + return response + redirected = self._redirect_request_using_get(request, url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response + if cast(float, interval) < self._maxdelay: return self._redirect(redirected, request, spider, "meta refresh") - return response diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..46587a898 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ + from __future__ import annotations import warnings diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..e7ecdbe0c 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ + from typing import Any # Internal diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..fb4998099 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats. """ import csv -import io import marshal -import pickle +import pickle # nosec import pprint -from collections.abc import Mapping -from xml.sax.saxutils import XMLGenerator +from io import BytesIO, TextIOWrapper +from json import JSONEncoder +from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union +from xml.sax.saxutils import XMLGenerator # nosec +from xml.sax.xmlreader import AttributesImpl # nosec from itemadapter import ItemAdapter, is_item -from scrapy.item import Item +from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -29,36 +31,42 @@ __all__ = [ class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): - self._kwargs = kwargs + def __init__(self, *, dont_fail: bool = False, **kwargs: Any): + self._kwargs: Dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop("encoding", None) - self.fields_to_export = options.pop("fields_to_export", None) - self.export_empty_fields = options.pop("export_empty_fields", False) - self.indent = options.pop("indent", None) + self.encoding: Optional[str] = options.pop("encoding", None) + self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( + options.pop("fields_to_export", None) + ) + self.export_empty_fields: bool = options.pop("export_empty_fields", False) + self.indent: Optional[int] = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") - def export_item(self, item): + def export_item(self, item: Any) -> None: raise NotImplementedError - def serialize_field(self, field, name, value): - serializer = field.get("serializer", lambda x: x) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self): + def start_exporting(self) -> None: pass - def finish_exporting(self): + def finish_exporting(self) -> None: pass - def _get_serialized_fields(self, item, default_value=None, include_empty=None): + def _get_serialized_fields( + self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + ) -> Iterable[Tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -100,22 +108,22 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file self._kwargs.setdefault("ensure_ascii", not self.encoding) - self.encoder = ScrapyJSONEncoder(**self._kwargs) + self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere @@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter): self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True - def _beautify_newline(self): + def _beautify_newline(self) -> None: if self.indent is not None: self.file.write(b"\n") - def _add_comma_after_first(self): + def _add_comma_after_first(self) -> None: if self.first_item: self.first_item = False else: self.file.write(b",") self._beautify_newline() - def start_exporting(self): + def start_exporting(self) -> None: self.file.write(b"[") self._beautify_newline() - def finish_exporting(self): + def finish_exporting(self) -> None: self._beautify_newline() self.file.write(b"]") - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() @@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): self.item_element = kwargs.pop("item_element", "item") self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) @@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter): self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self, new_item=False): + def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): self.xg.characters("\n") - def _beautify_indent(self, depth=1): + def _beautify_indent(self, depth: int = 1) -> None: if self.indent: self.xg.characters(" " * self.indent * depth) - def start_exporting(self): + def start_exporting(self) -> None: self.xg.startDocument() - self.xg.startElement(self.root_element, {}) + self.xg.startElement(self.root_element, AttributesImpl({})) self._beautify_newline(new_item=True) - def export_item(self, item): + def export_item(self, item: Any) -> None: self._beautify_indent(depth=1) - self.xg.startElement(self.item_element, {}) + self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) @@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(self.item_element) self._beautify_newline(new_item=True) - def finish_exporting(self): + def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value, depth): + def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) - self.xg.startElement(name, {}) + self.xg.startElement(name, AttributesImpl({})) if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): @@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): def __init__( self, - file, - include_headers_line=True, - join_multivalued=",", - errors=None, - **kwargs, + file: BytesIO, + include_headers_line: bool = True, + join_multivalued: str = ",", + errors: Optional[str] = None, + **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = "utf-8" self.include_headers_line = include_headers_line - self.stream = io.TextIOWrapper( + self.stream = TextIOWrapper( file, line_buffering=False, write_through=True, @@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._join_if_needed) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) - def _join_if_needed(self, value): + def _join_if_needed(self, value: Any) -> Any: if isinstance(value, (list, tuple)): try: return self._join_multivalued.join(value) @@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter): pass return value - def export_item(self, item): + def export_item(self, item: Any) -> None: if self._headers_not_written: self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) @@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) - def finish_exporting(self): + def finish_exporting(self) -> None: self.stream.detach() # Avoid closing the wrapped file. - def _build_row(self, values): + def _build_row(self, values: Iterable[Any]) -> Iterable[Any]: for s in values: try: yield to_unicode(s, self.encoding) except TypeError: yield s - def _write_headers_and_set_fields_to_export(self, item): + def _write_headers_and_set_fields_to_export(self, item: Any) -> None: if self.include_headers_line: if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() else: + assert self.fields_to_export fields = self.fields_to_export row = list(self._build_row(fields)) self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): + def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any): super().__init__(**kwargs) - self.file = file - self.protocol = protocol + self.file: BytesIO = file + self.protocol: int = protocol - def export_item(self, item): + def export_item(self, item: Any) -> None: d = dict(self._get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter): opened in binary mode, a :class:`~io.BytesIO` object, etc) """ - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: marshal.dump(dict(self._get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._serialize_value) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get( + "serializer", self._serialize_value + ) return serializer(value) - def _serialize_value(self, value): + def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) if is_item(value): @@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item): + def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item): - result = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] + result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..8221b675e 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,7 +3,11 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + +from typing import Any, List + from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list @@ -11,5 +15,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4307b4170..812b3553c 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,20 +4,31 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging from collections import defaultdict +from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from scrapy import signals +from twisted.python.failure import Failure + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class CloseSpider: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler - self.close_on = { + self.close_on: Dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -28,7 +39,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter = defaultdict(int) + self.counter: DefaultDict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) @@ -39,8 +50,8 @@ class CloseSpider: if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): - self.timeout_no_item = self.close_on["timeout_no_item"] - self.items_in_period = 0 + self.timeout_no_item: int = self.close_on["timeout_no_item"] + self.items_in_period: int = 0 crawler.signals.connect( self.spider_opened_no_item, signal=signals.spider_opened ) @@ -50,22 +61,25 @@ class CloseSpider: crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def error_count(self, failure, response, spider): + def error_count(self, failure: Failure, response: Response, spider: Spider) -> None: self.counter["errorcount"] += 1 if self.counter["errorcount"] == self.close_on["errorcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_errorcount") - def page_count(self, response, request, spider): + def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor + assert self.crawler.engine self.task = reactor.callLater( self.close_on["timeout"], self.crawler.engine.close_spider, @@ -73,21 +87,22 @@ class CloseSpider: reason="closespider_timeout", ) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 if self.counter["itemcount"] == self.close_on["itemcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") - def spider_closed(self, spider): - task = getattr(self, "task", False) + def spider_closed(self, spider: Spider) -> None: + task = getattr(self, "task", None) if task and task.active(): task.cancel() - task_no_item = getattr(self, "task_no_item", False) + task_no_item = getattr(self, "task_no_item", None) if task_no_item and task_no_item.running: task_no_item.stop() - def spider_opened_no_item(self, spider): + def spider_opened_no_item(self, spider: Spider) -> None: from twisted.internet import task self.task_no_item = task.LoopingCall(self._count_items_produced, spider) @@ -98,10 +113,10 @@ class CloseSpider: f"{self.timeout_no_item} seconds." ) - def item_scraped_no_item(self, item, spider): + def item_scraped_no_item(self, item: Any, spider: Spider) -> None: self.items_in_period += 1 - def _count_items_produced(self, spider): + def _count_items_produced(self, spider: Spider) -> None: if self.items_in_period >= 1: self.items_in_period = 0 else: @@ -109,4 +124,5 @@ class CloseSpider: f"Closing spider since no items were produced in the last " f"{self.timeout_no_item} seconds." ) + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..f3ac19623 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,18 +1,29 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime, timezone -from scrapy import signals +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class CoreStats: - def __init__(self, stats): - self.stats = stats - self.start_time = None + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats + self.start_time: Optional[datetime] = None @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -21,11 +32,12 @@ class CoreStats: crawler.signals.connect(o.response_received, signal=signals.response_received) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: + assert self.start_time is not None finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() @@ -35,13 +47,13 @@ class CoreStats: self.stats.set_value("finish_time", finish_time, spider=spider) self.stats.set_value("finish_reason", reason, spider=spider) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.stats.inc_value("item_scraped_count", spider=spider) - def response_received(self, spider): + def response_received(self, spider: Spider) -> None: self.stats.inc_value("response_received_count", spider=spider) - def item_dropped(self, item, spider, exception): + def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None: reason = exception.__class__.__name__ self.stats.inc_value("item_dropped_count", spider=spider) self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 1b6c7777f..26726b662 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,22 +4,31 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import signal import sys import threading import traceback from pdb import Pdb +from types import FrameType +from typing import TYPE_CHECKING, Optional +from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler try: signal.signal(signal.SIGUSR2, self.dump_stacktrace) signal.signal(signal.SIGQUIT, self.dump_stacktrace) @@ -28,10 +37,11 @@ class StackTraceDump: pass @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum, frame): + def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), @@ -44,7 +54,7 @@ class StackTraceDump: extra={"crawler": self.crawler}, ) - def _thread_stacks(self): + def _thread_stacks(self) -> str: id2name = dict((th.ident, th.name) for th in threading.enumerate()) dumps = "" for id_, frame in sys._current_frames().items(): @@ -55,12 +65,13 @@ class StackTraceDump: class Debugger: - def __init__(self): + def __init__(self) -> None: try: signal.signal(signal.SIGUSR2, self._enter_debugger) except AttributeError: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum, frame): + def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + assert frame Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fadbbb582..97f39afe7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,6 +4,8 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +from __future__ import annotations + import logging import re import sys @@ -11,26 +13,51 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import unquote, urlparse -from twisted.internet import defer, threads -from twisted.internet.defer import DeferredList +from twisted.internet import threads +from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager +from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode + + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) try: @@ -40,8 +67,19 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False +UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] -def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): +_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") + + +def build_storage( + builder: Callable[..., _StorageT], + uri: str, + *args: Any, + feed_options: Optional[Dict[str, Any]] = None, + preargs: Iterable[Any] = (), + **kwargs: Any, +) -> _StorageT: kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -55,10 +93,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict] - item_classes: Tuple + feed_options: Optional[Dict[str, Any]] + item_classes: Tuple[type, ...] - def __init__(self, feed_options: Optional[dict]) -> None: + def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -97,28 +135,49 @@ class IFeedStorage(Interface): """Store the given file stream""" +class FeedStorageProtocol(Protocol): + """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" + + def open(self, spider: Spider) -> IO[bytes]: + """Open the storage for the given spider. It must return a file-like + object that will be used for the exporters""" + + def store(self, file: IO[bytes]) -> Optional[Deferred]: + """Store the given file stream""" + + @implementer(IFeedStorage) class BlockingFeedStorage: - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): raise OSError("Not a Directory: " + str(path)) return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: return threads.deferToThread(self._store_in_thread, file) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): + def __init__( + self, + uri: str, + _stdout: Optional[IO[bytes]] = None, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): if not _stdout: _stdout = sys.stdout.buffer - self._stdout = _stdout + self._stdout: IO[bytes] = _stdout if feed_options and feed_options.get("overwrite", False) is True: logger.warning( "Standard output (stdout) storage does not support " @@ -127,54 +186,58 @@ class StdoutFeedStorage: "it to False." ) - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): - self.path = file_uri_to_path(uri) + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} - self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" + self.write_mode: OpenBinaryMode = ( + "wb" if feed_options.get("overwrite", False) else "ab" + ) - def open(self, spider) -> IO[Any]: + def open(self, spider: Spider) -> IO[bytes]: dirname = Path(self.path).parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: file.close() + return None class S3FeedStorage(BlockingFeedStorage): def __init__( self, - uri, - access_key=None, - secret_key=None, - acl=None, - endpoint_url=None, + uri: str, + access_key: Optional[str] = None, + secret_key: Optional[str] = None, + acl: Optional[str] = None, + endpoint_url: Optional[str] = None, *, - feed_options=None, - session_token=None, - region_name=None, + feed_options: Optional[Dict[str, Any]] = None, + session_token: Optional[str] = None, + region_name: Optional[str] = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) - self.bucketname = u.hostname - self.access_key = u.username or access_key - self.secret_key = u.password or secret_key - self.session_token = session_token - self.keyname = u.path[1:] # remove first "/" - self.acl = acl - self.endpoint_url = endpoint_url - self.region_name = region_name + assert u.hostname + self.bucketname: str = u.hostname + self.access_key: Optional[str] = u.username or access_key + self.secret_key: Optional[str] = u.password or secret_key + self.session_token: Optional[str] = session_token + self.keyname: str = u.path[1:] # remove first "/" + self.acl: Optional[str] = acl + self.endpoint_url: Optional[str] = endpoint_url + self.region_name: Optional[str] = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -217,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage): ) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -230,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) + kwargs: Dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -246,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): - self.project_id = project_id - self.acl = acl + def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): + self.project_id: Optional[str] = project_id + self.acl: Optional[str] = acl u = urlparse(uri) - self.bucket_name = u.hostname - self.blob_name = u.path[1:] # remove first "/" + assert u.hostname + self.bucket_name: str = u.hostname + self.blob_name: str = u.path[1:] # remove first "/" @classmethod - def from_crawler(cls, crawler, uri): + def from_crawler(cls, crawler: Crawler, uri: str) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) from google.cloud.storage import Client @@ -291,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage): self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -299,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: ftp_store_file( path=self.path, file=file, @@ -315,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - storage, - uri, - format, - store_empty, - batch_id, - uri_template, - filter, - feed_options, - spider, - exporters, - settings, - crawler, + storage: FeedStorageProtocol, + uri: str, + format: str, + store_empty: bool, + batch_id: int, + uri_template: str, + filter: ItemFilter, + feed_options: Dict[str, Any], + spider: Spider, + exporters: Dict[str, Type[BaseItemExporter]], + settings: BaseSettings, + crawler: Crawler, ): - self.file = None - self.exporter = None - self.storage = storage + self.file: Optional[IO[bytes]] = None + self.exporter: Optional[BaseItemExporter] = None + self.storage: FeedStorageProtocol = storage # feed params - self.batch_id = batch_id - self.format = format - self.store_empty = store_empty - self.uri_template = uri_template - self.uri = uri - self.filter = filter + self.batch_id: int = batch_id + self.format: str = format + self.store_empty: bool = store_empty + self.uri_template: str = uri_template + self.uri: str = uri + self.filter: ItemFilter = filter # exporter params - self.feed_options = feed_options - self.spider = spider - self.exporters = exporters - self.settings = settings - self.crawler = crawler + self.feed_options: Dict[str, Any] = feed_options + self.spider: Spider = spider + self.exporters: Dict[str, Type[BaseItemExporter]] = exporters + self.settings: BaseSettings = settings + self.crawler: Crawler = crawler # flags - self.itemcount = 0 - self._exporting = False - self._fileloaded = False + self.itemcount: int = 0 + self._exporting: bool = False + self._fileloaded: bool = False - def start_exporting(self): + def start_exporting(self) -> None: if not self._fileloaded: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: - self.file = PostProcessingManager( - self.feed_options["postprocessing"], self.file, self.feed_options + self.file = cast( + IO[bytes], + PostProcessingManager( + self.feed_options["postprocessing"], + self.file, + self.feed_options, + ), ) self.exporter = self._get_exporter( file=self.file, @@ -367,17 +449,23 @@ class FeedSlot: self._fileloaded = True if not self._exporting: + assert self.exporter self.exporter.start_exporting() self._exporting = True - def _get_instance(self, objcls, *args, **kwargs): - return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + def _get_instance( + self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + ) -> BaseItemExporter: + return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs): + def _get_exporter( + self, file: IO[bytes], format: str, *args: Any, **kwargs: Any + ) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) - def finish_exporting(self): + def finish_exporting(self) -> None: if self._exporting: + assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -389,22 +477,22 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[defer.Deferred] = [] + _pending_deferreds: List[Deferred] = [] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: exporter = cls(crawler) crawler.signals.connect(exporter.open_spider, signals.spider_opened) crawler.signals.connect(exporter.close_spider, signals.spider_closed) crawler.signals.connect(exporter.item_scraped, signals.item_scraped) return exporter - def __init__(self, crawler): - self.crawler = crawler - self.settings = crawler.settings + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.settings: Settings = crawler.settings self.feeds = {} - self.slots = [] - self.filters = {} + self.slots: List[FeedSlot] = [] + self.filters: Dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -436,8 +524,12 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components("FEED_STORAGES") - self.exporters = self._load_components("FEED_EXPORTERS") + self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + "FEED_STORAGES" + ) + self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + "FEED_EXPORTERS" + ) for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured @@ -446,7 +538,7 @@ class FeedExporter: if not self._exporter_supported(feed_options["format"]): raise NotConfigured - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: for uri, feed_options in self.feeds.items(): uri_params = self._get_uri_params(spider, feed_options["uri_params"]) self.slots.append( @@ -459,7 +551,7 @@ class FeedExporter: ) ) - async def close_spider(self, spider): + async def close_spider(self, spider: Spider) -> None: for slot in self.slots: self._close_slot(slot, spider) @@ -472,8 +564,9 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot, spider): - def get_file(slot_): + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file if isinstance(slot_.file, PostProcessingManager): slot_.file.close() return slot_.file.file @@ -491,7 +584,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -509,20 +602,33 @@ class FeedExporter: return d - def _handle_store_error(self, f, logmsg, spider, slot_type): + def _handle_store_error( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.error( "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, logmsg, spider, slot_type): + def _handle_store_success( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") - def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + def _start_new_batch( + self, + batch_id: int, + uri: str, + feed_options: Dict[str, Any], + spider: Spider, + uri_template: str, + ) -> FeedSlot: """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified @@ -545,11 +651,11 @@ class FeedExporter: spider=spider, exporters=self.exporters, settings=self.settings, - crawler=getattr(self, "crawler", None), + crawler=self.crawler, ) return slot - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] for slot in self.slots: if not slot.filter.accepts(item): @@ -559,6 +665,7 @@ class FeedExporter: continue slot.start_exporting() + assert slot.exporter slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one @@ -583,7 +690,7 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix): + def _load_components(self, setting_prefix: str) -> Dict[str, Any]: conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): @@ -593,12 +700,13 @@ class FeedExporter: pass return d - def _exporter_supported(self, format): + def _exporter_supported(self, format: str) -> bool: if format in self.exporters: return True logger.error("Unknown feed format: %(format)s", {"format": format}) + return False - def _settings_are_valid(self): + def _settings_are_valid(self) -> bool: """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)d to distinguish different files of partial output @@ -616,7 +724,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri, feed_options): + def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -629,8 +737,11 @@ class FeedExporter: ) else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) + return False - def _get_storage(self, uri, feed_options): + def _get_storage( + self, uri: str, feed_options: Dict[str, Any] + ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that @@ -639,11 +750,14 @@ class FeedExporter: feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) - def build_instance(builder, *preargs): + def build_instance( + builder: Type[FeedStorageProtocol], *preargs: Any + ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs ) + instance: FeedStorageProtocol if crawler and hasattr(feedcls, "from_crawler"): instance = build_instance(feedcls.from_crawler, crawler) method_name = "from_crawler" @@ -660,9 +774,9 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> dict: + ) -> Dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -670,7 +784,7 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = ( + uripar_function: UriParamsCallableT = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params @@ -678,7 +792,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options): + def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + item_filter_class: Type[ItemFilter] = load_object( + feed_options.get("item_filter", ItemFilter) + ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7e4f047a8..dd5bce24f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,10 +1,13 @@ import gzip import logging -import pickle +import os +import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time +from types import ModuleType +from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.request import RequestFingerprinter + +if TYPE_CHECKING: + # typing.Concatenate requires Python 3.10 + from typing_extensions import Concatenate + logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes = [ + def __init__(self, settings: BaseSettings): + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: List[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: return urlparse_cached(request).scheme not in self.ignore_schemes - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: return True - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: return True class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year - def __init__(self, settings): - self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self._cc_parsed = WeakKeyDictionary() - self.ignore_response_cache_controls = [ + def __init__(self, settings: BaseSettings): + self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self._cc_parsed: WeakKeyDictionary[ + Union[Request, Response], Dict[bytes, Optional[bytes]] + ] = WeakKeyDictionary() + self.ignore_response_cache_controls: List[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol(self, r): + def _parse_cachecontrol( + self, r: Union[Request, Response] + ) -> Dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") + assert cch is not None parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -62,7 +81,7 @@ class RFC2616Policy: self._cc_parsed[r] = parsed return self._cc_parsed[r] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: if urlparse_cached(request).scheme in self.ignore_schemes: return False cc = self._parse_cachecontrol(request) @@ -72,7 +91,7 @@ class RFC2616Policy: # Any other is eligible for caching return True - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents @@ -100,7 +119,9 @@ class RFC2616Policy: # info and can not be revalidated return False - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) if b"no-cache" in cc or b"no-cache" in ccreq: @@ -141,7 +162,9 @@ class RFC2616Policy: self._set_conditional_validators(request, cachedresponse) return False - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: # Use the cached response if the new response is a server error, # as long as the old response didn't specify must-revalidate. if response.status >= 500: @@ -152,7 +175,9 @@ class RFC2616Policy: # Use the cached response if the server says it hasn't changed. return response.status == 304 - def _set_conditional_validators(self, request, cachedresponse): + def _set_conditional_validators( + self, request: Request, cachedresponse: Response + ) -> None: if b"Last-Modified" in cachedresponse.headers: request.headers[b"If-Modified-Since"] = cachedresponse.headers[ b"Last-Modified" @@ -161,13 +186,15 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc): + def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: try: - return max(0, int(cc[b"max-age"])) + return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): return None - def _compute_freshness_lifetime(self, response, request, now): + def _compute_freshness_lifetime( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeFreshnessLifetime # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) @@ -198,10 +225,12 @@ class RFC2616Policy: # Insufficient information to compute freshness lifetime return 0 - def _compute_current_age(self, response, request, now): + def _compute_current_age( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeCurrentAge # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 - currentage = 0 + currentage: float = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server date = rfc1123_to_epoch(response.headers.get(b"Date")) or now @@ -210,7 +239,7 @@ class RFC2616Policy: if b"Age" in response.headers: try: - age = int(response.headers[b"Age"]) + age = int(response.headers[b"Age"]) # type: ignore[arg-type] currentage = max(currentage, age) except ValueError: pass @@ -219,13 +248,13 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) - self.db = None + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) + self.db: Any = None # the real type is private - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: dbpath = Path(self.cachedir, f"{spider.name}.db") self.db = self.dbmodule.open(str(dbpath), "c") @@ -235,15 +264,16 @@ class DbmCacheStorage: extra={"spider": spider}, ) - self._fingerprinter = spider.crawler.request_fingerprinter + assert spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: data = self._read_data(spider, request) if data is None: - return # not cached + return None # not cached url = data["url"] status = data["status"] headers = Headers(data["headers"]) @@ -252,7 +282,9 @@ class DbmCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: key = self._fingerprinter.fingerprint(request).hex() data = { "status": response.status, @@ -263,28 +295,31 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider, request): + def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" if tkey not in db: - return # not found + return None # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return None # expired - return pickle.loads(db[f"{key}_data"]) + return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"]) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.use_gzip = settings.getbool("HTTPCACHE_GZIP") - self._open = gzip.open if self.use_gzip else open + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + # https://github.com/python/mypy/issues/10740 + self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + gzip.open if self.use_gzip else open # type: ignore[assignment] + ) - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: logger.debug( "Using filesystem cache storage in %(cachedir)s", {"cachedir": self.cachedir}, @@ -294,27 +329,29 @@ class FilesystemCacheStorage: assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: - return # not cached + return None # not cached rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / "response_body", "rb") as f: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get("response_url") + url = metadata["response_url"] status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider: Spider, request: Request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: """Store the given response in the cache.""" rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): @@ -343,19 +380,19 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request): + def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): - return # not found + return None # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: - return # expired + return None # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return cast(Dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header): +def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -375,9 +412,9 @@ def parse_cachecontrol(header): return directives -def rfc1123_to_epoch(date_str): +def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: try: - date_str = to_unicode(date_str, encoding="ascii") - return mktime_tz(parsedate_tz(date_str)) + date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] + return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] except Exception: return None diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 78874a6db..2388afa75 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,58 +1,94 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class LogStats: - """Log basic scraping stats periodically""" + """Log basic scraping stats periodically like: + * RPM - Requests per Minute + * IPM - Items per Minute + """ - def __init__(self, stats, interval=60.0): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None + def __init__(self, stats: StatsCollector, interval: float = 60.0): + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured + assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.pagesprev = 0 - self.itemsprev = 0 + def spider_opened(self, spider: Spider) -> None: + self.pagesprev: int = 0 + self.itemsprev: int = 0 self.task = task.LoopingCall(self.log, spider) self.task.start(self.interval) - def log(self, spider): - items = self.stats.get_value("item_scraped_count", 0) - pages = self.stats.get_value("response_received_count", 0) - irate = (items - self.itemsprev) * self.multiplier - prate = (pages - self.pagesprev) * self.multiplier - self.pagesprev, self.itemsprev = pages, items + def log(self, spider: Spider) -> None: + self.calculate_stats() msg = ( "Crawled %(pages)d pages (at %(pagerate)d pages/min), " "scraped %(items)d items (at %(itemrate)d items/min)" ) log_args = { - "pages": pages, - "pagerate": prate, - "items": items, - "itemrate": irate, + "pages": self.pages, + "pagerate": self.prate, + "items": self.items, + "itemrate": self.irate, } logger.info(msg, log_args, extra={"spider": spider}) - def spider_closed(self, spider, reason): + def calculate_stats(self) -> None: + self.items: int = self.stats.get_value("item_scraped_count", 0) + self.pages: int = self.stats.get_value("response_received_count", 0) + self.irate: float = (self.items - self.itemsprev) * self.multiplier + self.prate: float = (self.pages - self.pagesprev) * self.multiplier + self.pagesprev, self.itemsprev = self.pages, self.items + + def spider_closed(self, spider: Spider, reason: str) -> None: if self.task and self.task.running: self.task.stop() + + rpm_final, ipm_final = self.calculate_final_stats(spider) + self.stats.set_value("responses_per_minute", rpm_final) + self.stats.set_value("items_per_minute", ipm_final) + + def calculate_final_stats( + self, spider: Spider + ) -> Union[Tuple[None, None], Tuple[float, float]]: + start_time = self.stats.get_value("start_time") + finished_time = self.stats.get_value("finished_time") + + if not start_time or not finished_time: + return None, None + + mins_elapsed = (finished_time - start_time).seconds / 60 + + items = self.stats.get_value("item_scraped_count", 0) + pages = self.stats.get_value("response_received_count", 0) + + return (pages / mins_elapsed), (items / mins_elapsed) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 03ede0681..f304e1bf2 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -4,26 +4,36 @@ MemoryDebugger extension See documentation in docs/topics/extensions.rst """ -import gc +from __future__ import annotations -from scrapy import signals +import gc +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryDebugger: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: gc.collect() self.stats.set_value( "memdebug/gc_garbage_count", len(gc.garbage), spider=spider diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 221967bda..9de06b24d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,24 +3,33 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + +from __future__ import annotations + import logging import socket import sys from importlib import import_module from pprint import pformat +from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: @@ -29,32 +38,33 @@ class MemoryUsage: except ImportError: raise NotConfigured - self.crawler = crawler - self.warned = False - self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") - self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 - self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 - self.check_interval = crawler.settings.getfloat( + self.crawler: Crawler = crawler + self.warned: bool = False + self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail = MailSender.from_settings(crawler.settings) + self.mail: MailSender = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def get_virtual_size(self): - size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss + def get_virtual_size(self) -> int: + size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size - def engine_started(self): + def engine_started(self) -> None: + assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks = [] + self.tasks: List[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -67,15 +77,18 @@ class MemoryUsage: self.tasks.append(tsk) tsk.start(self.check_interval, now=True) - def engine_stopped(self): + def engine_stopped(self) -> None: for tsk in self.tasks: if tsk.running: tsk.stop() - def update(self): + def update(self) -> None: + assert self.crawler.stats self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) - def _check_limit(self): + def _check_limit(self) -> None: + assert self.crawler.engine + assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: self.crawler.stats.set_value("memusage/limit_reached", 1) @@ -105,9 +118,10 @@ class MemoryUsage: {"virtualsize": peak_mem_usage / 1024 / 1024}, ) - def _check_warning(self): + def _check_warning(self) -> None: if self.warned: # warn only once return + assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) mem = self.warning / 1024 / 1024 @@ -125,12 +139,14 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts, subject): + def _send_report(self, rcpts: List[str], subject: str) -> None: """send notification mail with some additional useful info""" + assert self.crawler.engine + assert self.crawler.stats stats = self.crawler.stats - s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" - s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" - s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n" s += ( "ENGINE STATUS ------------------------------------------------------- \r\n" diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 2d557f123..9567f948a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import logging from datetime import datetime, timezone +from json import JSONEncoder +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -15,32 +25,34 @@ class PeriodicLog: def __init__( self, - stats, - interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + stats: StatsCollector, + interval: float = 60.0, + ext_stats: Dict[str, Any] = {}, + ext_delta: Dict[str, Any] = {}, + ext_timing_enabled: bool = False, ): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None - self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = bool(ext_stats) - self.ext_stats_include = ext_stats.get("include", []) - self.ext_stats_exclude = ext_stats.get("exclude", []) - self.ext_delta_enabled = bool(ext_delta) - self.ext_delta_include = ext_delta.get("include", []) - self.ext_delta_exclude = ext_delta.get("exclude", []) - self.ext_timing_enabled = ext_timing_enabled + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None + self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled: bool = bool(ext_stats) + self.ext_stats_include: List[str] = ext_stats.get("include", []) + self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_delta_enabled: bool = bool(ext_delta) + self.ext_delta_include: List[str] = ext_delta.get("include", []) + self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_timing_enabled: bool = ext_timing_enabled @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured try: - ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_STATS" + ) except (TypeError, ValueError): ext_stats = ( {"enabled": True} @@ -48,7 +60,9 @@ class PeriodicLog: else None ) try: - ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_DELTA" + ) except (TypeError, ValueError): ext_delta = ( {"enabled": True} @@ -56,11 +70,14 @@ class PeriodicLog: else None ) - ext_timing_enabled = crawler.settings.getbool( + ext_timing_enabled: bool = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured + assert crawler.stats + assert ext_stats is not None + assert ext_delta is not None o = cls( crawler.stats, interval, @@ -72,16 +89,16 @@ class PeriodicLog: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.time_prev = datetime.now(tz=timezone.utc) - self.delta_prev = {} - self.stats_prev = {} + def spider_opened(self, spider: Spider) -> None: + self.time_prev: datetime = datetime.now(tz=timezone.utc) + self.delta_prev: Dict[str, Union[int, float]] = {} + self.stats_prev: Dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) - def log(self): - data = {} + def log(self) -> None: + data: Dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -90,8 +107,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): - num_stats = { + def log_delta(self) -> Dict[str, Any]: + num_stats: Dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -101,7 +118,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self): + def log_timing(self) -> Dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -113,7 +130,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self): + def log_crawler_stats(self) -> Dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -121,7 +138,9 @@ class PeriodicLog: } return {"stats": stats} - def param_allowed(self, stat_name, include, exclude): + def param_allowed( + self, stat_name: str, include: List[str], exclude: List[str] + ) -> bool: if not include and not exclude: return True for p in exclude: @@ -134,7 +153,7 @@ class PeriodicLog: return True return False - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: self.log() if self.task and self.task.running: self.task.stop() diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 79e3b1656..ac12ad829 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,11 +1,12 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List +from typing import IO, Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -42,7 +43,6 @@ class GzipPlugin: def close(self) -> None: self.gzipfile.close() - self.file.close() class Bz2Plugin: @@ -69,7 +69,6 @@ class Bz2Plugin: def close(self) -> None: self.bz2file.close() - self.file.close() class LZMAPlugin: @@ -111,7 +110,6 @@ class LZMAPlugin: def close(self) -> None: self.lzmafile.close() - self.file.close() # io.IOBase is subclassed here, so that exporters can use the PostProcessingManager @@ -128,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file @@ -144,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return self.head_plugin.write(data) + return cast(int, self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..c6eb20277 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,19 +1,27 @@ -import pickle -from pathlib import Path +from __future__ import annotations -from scrapy import signals +import pickle # nosec +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir=None): - self.jobdir = jobdir + def __init__(self, jobdir: Optional[str] = None): + self.jobdir: Optional[str] = jobdir @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: jobdir = job_dir(crawler.settings) if not jobdir: raise NotConfigured @@ -23,18 +31,20 @@ class SpiderState: crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: if self.jobdir: with Path(self.statefn).open("wb") as f: + assert hasattr(spider, "state") # set in spider_opened pickle.dump(spider.state, f, protocol=4) - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec else: - spider.state = {} + spider.state = {} # type: ignore[attr-defined] @property def statefn(self) -> str: + assert self.jobdir return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 58610c25e..20b8f910c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +from twisted.internet.defer import Deferred + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class StatsMailer: - def __init__(self, stats, recipients, mail): - self.stats = stats - self.recipients = recipients - self.mail = mail + def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + self.stats: StatsCollector = stats + self.recipients: List[str] = recipients + self.mail: MailSender = mail @classmethod - def from_crawler(cls, crawler): - recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + def from_crawler(cls, crawler: Crawler) -> Self: + recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_settings(crawler.settings) + assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> Optional[Deferred]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c92b7f5fe..00c69434c 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,13 +4,17 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ +from __future__ import annotations + import binascii import logging import os import pprint import traceback +from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol +from twisted.internet.tcp import Port try: from twisted.conch import manhole, telnet @@ -22,12 +26,16 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) # signal to update telnet variables @@ -36,7 +44,7 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: @@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory): "TELNETCONSOLE_ENABLED setting is True but required twisted " "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK ) - self.crawler = crawler - self.noisy = False - self.portrange = [ + self.crawler: Crawler = crawler + self.noisy: bool = False + self.portrange: List[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] - self.host = crawler.settings["TELNETCONSOLE_HOST"] - self.username = crawler.settings["TELNETCONSOLE_USERNAME"] - self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] + self.host: str = crawler.settings["TELNETCONSOLE_HOST"] + self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode("utf8") @@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory): self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) + def start_listening(self) -> None: + self.port: Port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory): extra={"crawler": self.crawler}, ) - def stop_listening(self): + def stop_listening(self) -> None: self.port.stopListening() - def protocol(self): + def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] class Portal: """An implementation of IPortal""" @@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self): + def _get_telnet_vars(self) -> Dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { + assert self.crawler.engine + telnet_vars: Dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 396800775..217e61a81 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,51 +1,68 @@ -import logging +from __future__ import annotations -from scrapy import signals +import logging +from typing import TYPE_CHECKING, Optional, Tuple + +from scrapy import Request, Spider, signals +from scrapy.core.downloader import Slot +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured - self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat( + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) + if self.target_concurrency <= 0.0: + raise NotConfigured( + f"AUTOTHROTTLE_TARGET_CONCURRENCY " + f"({self.target_concurrency!r}) must be higher than 0." + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def _spider_opened(self, spider): + def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) + spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] - def _min_delay(self, spider): + def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) - def _max_delay(self, spider): + def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider): + def _start_delay(self, spider: Spider) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) - def _response_downloaded(self, response, request, spider): + def _response_downloaded( + self, response: Response, request: Request, spider: Spider + ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if latency is None or slot is None or slot.throttle is False: return olddelay = slot.delay @@ -69,11 +86,16 @@ class AutoThrottle: extra={"spider": spider}, ) - def _get_slot(self, request, spider): - key = request.meta.get("download_slot") + def _get_slot( + self, request: Request, spider: Spider + ) -> Tuple[Optional[str], Optional[Slot]]: + key: Optional[str] = request.meta.get("download_slot") + if key is None: + return None, None + assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) - def _adjust_delay(self, slot, latency, response): + def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: """Define delay adjustment policy""" # If a server needs `latency` seconds to respond then diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index ac3946302..d0b726bad 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse +from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 2595f328e..8af89c74f 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,36 +1,56 @@ +from __future__ import annotations + import re import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import DefaultCookiePolicy -from typing import Sequence +from http.cookiejar import CookiePolicy, DefaultCookiePolicy +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + cast, +) from scrapy import Request from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: - def __init__(self, policy=None, check_expired_frequency=10000): - self.policy = policy or DefaultCookiePolicy() - self.jar = _CookieJar(self.policy) - self.jar._cookies_lock = _DummyLock() - self.check_expired_frequency = check_expired_frequency - self.processed = 0 + def __init__( + self, + policy: Optional[CookiePolicy] = None, + check_expired_frequency: int = 10000, + ): + self.policy: CookiePolicy = policy or DefaultCookiePolicy() + self.jar: _CookieJar = _CookieJar(self.policy) + self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined] + self.check_expired_frequency: int = check_expired_frequency + self.processed: int = 0 - def extract_cookies(self, response, request): + def extract_cookies(self, response: Response, request: Request) -> None: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.extract_cookies(wrsp, wreq) + self.jar.extract_cookies(wrsp, wreq) # type: ignore[arg-type] def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) - self.policy._now = self.jar._now = int(time.time()) + self.policy._now = self.jar._now = int(time.time()) # type: ignore[attr-defined] # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain @@ -47,10 +67,10 @@ class CookieJar: cookies = [] for host in hosts: - if host in self.jar._cookies: - cookies += self.jar._cookies_for_domain(host, wreq) + if host in self.jar._cookies: # type: ignore[attr-defined] + cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] - attrs = self.jar._cookie_attrs(cookies) + attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] if attrs: if not wreq.has_header("Cookie"): wreq.add_unredirected_header("Cookie", "; ".join(attrs)) @@ -61,37 +81,42 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self): - return self.jar._cookies + def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + return self.jar._cookies # type: ignore[attr-defined,no-any-return] - def clear_session_cookies(self, *args, **kwargs): - return self.jar.clear_session_cookies(*args, **kwargs) + def clear_session_cookies(self) -> None: + return self.jar.clear_session_cookies() - def clear(self, domain=None, path=None, name=None): - return self.jar.clear(domain, path, name) + def clear( + self, + domain: Optional[str] = None, + path: Optional[str] = None, + name: Optional[str] = None, + ) -> None: + self.jar.clear(domain, path, name) - def __iter__(self): + def __iter__(self) -> Iterator[Cookie]: return iter(self.jar) - def __len__(self): + def __len__(self) -> int: return len(self.jar) - def set_policy(self, pol): - return self.jar.set_policy(pol) + def set_policy(self, pol: CookiePolicy) -> None: + self.jar.set_policy(pol) def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.make_cookies(wrsp, wreq) + return self.jar.make_cookies(wrsp, wreq) # type: ignore[arg-type] - def set_cookie(self, cookie): + def set_cookie(self, cookie: Cookie) -> None: self.jar.set_cookie(cookie) def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: - self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) + self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain): +def potential_domain_matches(domain: str) -> List[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -111,10 +136,10 @@ def potential_domain_matches(domain): class _DummyLock: - def acquire(self): + def acquire(self) -> None: pass - def release(self): + def release(self) -> None: pass @@ -124,19 +149,19 @@ class WrappedRequest: see http://docs.python.org/library/urllib2.html#urllib2.Request """ - def __init__(self, request): + def __init__(self, request: Request): self.request = request - def get_full_url(self): + def get_full_url(self) -> str: return self.request.url - def get_host(self): + def get_host(self) -> str: return urlparse_cached(self.request).netloc - def get_type(self): + def get_type(self) -> str: return urlparse_cached(self.request).scheme - def is_unverifiable(self): + def is_unverifiable(self) -> bool: """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. It defaults to False. An unverifiable request is one whose URL the user did not have the @@ -144,35 +169,36 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return self.request.meta.get("is_unverifiable", False) + return cast(bool, self.request.meta.get("is_unverifiable", False)) @property - def full_url(self): + def full_url(self) -> str: return self.get_full_url() @property - def host(self): + def host(self) -> str: return self.get_host() @property - def type(self): + def type(self) -> str: return self.get_type() @property - def unverifiable(self): + def unverifiable(self) -> bool: return self.is_unverifiable() @property - def origin_req_host(self): - return urlparse_cached(self.request).hostname + def origin_req_host(self) -> str: + return cast(str, urlparse_cached(self.request).hostname) - def has_header(self, name): + def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name, default=None): - return to_unicode(self.request.headers.get(name, default), errors="replace") + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + value = self.request.headers.get(name, default) + return to_unicode(value, errors="replace") if value is not None else None - def header_items(self): + def header_items(self) -> List[Tuple[str, List[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -181,18 +207,18 @@ class WrappedRequest: for k, v in self.request.headers.items() ] - def add_unredirected_header(self, name, value): + def add_unredirected_header(self, name: str, value: str) -> None: self.request.headers.appendlist(name, value) class WrappedResponse: - def __init__(self, response): + def __init__(self, response: Response): self.response = response - def info(self): + def info(self) -> Self: return self - def get_all(self, name, default=None): + def get_all(self, name: str, default: Any = None) -> List[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..73aee7178 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -113,7 +113,9 @@ class Headers(CaselessDict): return ((k, self.getlist(k)) for k in self.keys()) def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + return [ + self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: # cast() can be removed if the headers_dict_to_raw() hint is improved diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..191b3cef4 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,8 +4,12 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -16,8 +20,6 @@ from typing import ( NoReturn, Optional, Tuple, - Type, - TypeVar, Union, cast, ) @@ -31,7 +33,9 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -185,11 +189,11 @@ class Request(object_ref): @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -231,12 +235,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0f80a0ab7..3206d79cd 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -10,21 +10,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import ( - FormElement, - HTMLParser, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) -from parsel.selector import create_root_node +from lxml.html import FormElement # nosec +from lxml.html import InputElement # nosec +from lxml.html import MultipleSelectOptions # nosec +from lxml.html import SelectElement # nosec +from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes -from scrapy.utils.response import get_base_url if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -120,7 +115,7 @@ def _get_form( formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" - root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) + root = response.selector.root forms = root.xpath("//form") if not forms: raise ValueError(f"No
element found in {response}") diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bde860a66..e20e7c438 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,12 +4,17 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + import xmlrpc.client as xmlrpclib from typing import Any, Optional +import defusedxml.xmlrpc + from scrapy.http.request import Request from scrapy.utils.python import get_func_args +defusedxml.xmlrpc.monkey_patch() + DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..d73dfce4b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations from ipaddress import IPv4Address, IPv6Address diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py new file mode 100644 index 000000000..219691094 --- /dev/null +++ b/scrapy/http/response/json.py @@ -0,0 +1,12 @@ +""" +This module implements the JsonResponse class that is used when the response +has a JSON MIME type in its Content-Type header. + +See documentation in docs/topics/request-response.rst +""" + +from scrapy.http.response.text import TextResponse + + +class JsonResponse(TextResponse): + pass diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6596d8a5c..522ffc0d5 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json @@ -158,12 +159,12 @@ class TextResponse(Response): def jmespath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList - if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + if not hasattr(self.selector, "jmespath"): raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] + return cast(SelectorList, self.selector.jmespath(query, **kwargs)) def xpath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList diff --git a/scrapy/item.py b/scrapy/item.py index d3eb90b7b..2daea64cc 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,14 +4,28 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from __future__ import annotations + from abc import ABCMeta -from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import Dict +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + KeysView, + MutableMapping, + NoReturn, + Tuple, +) from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class Field(dict): """Container of field metadata""" @@ -23,7 +37,9 @@ class ItemMeta(ABCMeta): .. _metaclass: https://realpython.com/python-metaclasses """ - def __new__(mcs, class_name, bases, attrs): + def __new__( + mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) @@ -44,7 +60,7 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class Item(MutableMapping, object_ref, metaclass=ItemMeta): +class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): """ Base class for scraped items. @@ -69,51 +85,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): fields: Dict[str, Field] - def __init__(self, *args, **kwargs): - self._values = {} + def __init__(self, *args: Any, **kwargs: Any): + self._values: Dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._values[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if key in self.fields: self._values[key] = value else: raise KeyError(f"{self.__class__.__name__} does not support field: {key}") - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._values[key] - def __getattr__(self, name): + def __getattr__(self, name: str) -> NoReturn: if name in self.fields: raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) - def __len__(self): + def __len__(self) -> int: return len(self._values) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._values) __hash__ = object_ref.__hash__ - def keys(self): + def keys(self) -> KeysView[str]: return self._values.keys() - def __repr__(self): + def __repr__(self) -> str: return pformat(dict(self)) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def deepcopy(self): + def deepcopy(self) -> Self: """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..4bdbc1823 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + from typing import Any diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..d59005edd 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,7 +5,9 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ + import re +from typing import Iterable, Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -109,14 +111,11 @@ IGNORED_EXTENSIONS = [ ] -_re_type = type(re.compile("", 0)) - - -def _matches(url, regexs): +def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) -def _is_valid_url(url): +def _is_valid_url(url: str) -> bool: return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..33a10cd6c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,24 +1,34 @@ """ Link extractor based on lxml.html """ + import logging import operator from functools import partial +from typing import ( + Any, + Callable, + Iterable, + List, + Optional, + Pattern, + Set, + Tuple, + Union, + cast, +) from urllib.parse import urljoin, urlparse -from lxml import etree +from lxml import etree # nosec +from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string +from scrapy import Selector +from scrapy.http import TextResponse from scrapy.link import Link -from scrapy.linkextractors import ( - IGNORED_EXTENSIONS, - _is_valid_url, - _matches, - _re_type, - re, -) +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url @@ -32,41 +42,56 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" _collect_string_content = etree.XPath("string()") -def _nons(tag): +def _nons(tag: Any) -> Any: if isinstance(tag, str): if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: return tag.split("}")[-1] return tag -def _identity(x): +def _identity(x: Any) -> Any: return x -def _canonicalize_link_url(link): +def _canonicalize_link_url(link: Link) -> str: return canonicalize_url(link.url, keep_fragments=True) class LxmlParserLinkExtractor: def __init__( self, - tag="a", - attr="href", - process=None, - unique=False, - strip=True, - canonicalized=False, + tag: Union[str, Callable[[str], bool]] = "a", + attr: Union[str, Callable[[str], bool]] = "href", + process: Optional[Callable[[Any], Any]] = None, + unique: bool = False, + strip: bool = True, + canonicalized: bool = False, ): - self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) - self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) - self.process_attr = process if callable(process) else _identity - self.unique = unique - self.strip = strip - self.link_key = ( - operator.attrgetter("url") if canonicalized else _canonicalize_link_url + # mypy doesn't infer types for operator.* and also for partial() + self.scan_tag: Callable[[str], bool] = ( + tag + if callable(tag) + else cast(Callable[[str], bool], partial(operator.eq, tag)) + ) + self.scan_attr: Callable[[str], bool] = ( + attr + if callable(attr) + else cast(Callable[[str], bool], partial(operator.eq, attr)) + ) + self.process_attr: Callable[[Any], Any] = ( + process if callable(process) else _identity + ) + self.unique: bool = unique + self.strip: bool = strip + self.link_key: Callable[[Link], str] = ( + cast(Callable[[Link], str], operator.attrgetter("url")) + if canonicalized + else _canonicalize_link_url ) - def _iter_links(self, document): + def _iter_links( + self, document: HtmlElement + ) -> Iterable[Tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -74,10 +99,16 @@ class LxmlParserLinkExtractor: for attrib in attribs: if not self.scan_attr(attrib): continue - yield (el, attrib, attribs[attrib]) + yield el, attrib, attribs[attrib] - def _extract_links(self, selector, response_url, response_encoding, base_url): - links = [] + def _extract_links( + self, + selector: Selector, + response_url: str, + response_encoding: str, + base_url: str, + ) -> List[Link]: + links: List[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -107,44 +138,48 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links): + def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: if self.unique: return unique_list(links, key=self.link_key) return links +_RegexT = Union[str, Pattern[str]] +_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] + + class LxmlLinkExtractor: _csstranslator = HTMLTranslator() def __init__( self, - allow=(), - deny=(), - allow_domains=(), - deny_domains=(), - restrict_xpaths=(), - tags=("a", "area"), - attrs=("href",), - canonicalize=False, - unique=True, - process_value=None, - deny_extensions=None, - restrict_css=(), - strip=True, - restrict_text=None, + allow: _RegexOrSeveralT = (), + deny: _RegexOrSeveralT = (), + allow_domains: Union[str, Iterable[str]] = (), + deny_domains: Union[str, Iterable[str]] = (), + restrict_xpaths: Union[str, Iterable[str]] = (), + tags: Union[str, Iterable[str]] = ("a", "area"), + attrs: Union[str, Iterable[str]] = ("href",), + canonicalize: bool = False, + unique: bool = True, + process_value: Optional[Callable[[Any], Any]] = None, + deny_extensions: Union[str, Iterable[str], None] = None, + restrict_css: Union[str, Iterable[str]] = (), + strip: bool = True, + restrict_text: Optional[_RegexOrSeveralT] = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -153,33 +188,33 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=canonicalize, + canonicalized=not canonicalize, ) - self.allow_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) - ] - self.deny_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny) - ] + self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) + self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) + self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS - self.canonicalize = canonicalize - self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [ - x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text) + self.canonicalize: bool = canonicalize + self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) + + @staticmethod + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: + return [ + x if isinstance(x, re.Pattern) else re.compile(x) + for x in arg_to_iter(value) ] - def _link_allowed(self, link): + def _link_allowed(self, link: Link) -> bool: if not _is_valid_url(link.url): return False if self.allow_res and not _matches(link.url, self.allow_res): @@ -201,7 +236,7 @@ class LxmlLinkExtractor: return False return True - def matches(self, url): + def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): @@ -215,7 +250,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -223,10 +258,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args, **kwargs): + def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. @@ -248,5 +283,5 @@ class LxmlLinkExtractor: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) if self.link_extractor.unique: - return unique_list(all_links) + return unique_list(all_links, key=self.link_extractor.link_key) return all_links diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..529fa279e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,6 +3,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ + import itemloaders from scrapy.item import Item diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..fd6302550 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,6 +3,9 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + +from __future__ import annotations + import logging from email import encoders as Encoders from email.mime.base import MIMEBase @@ -11,14 +14,36 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) from twisted import version as twisted_version -from twisted.internet import defer, ssl +from twisted.internet import ssl +from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.python.versions import Version +from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory + + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -27,7 +52,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text): +def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: if text is None: return None return to_bytes(text) @@ -36,26 +61,26 @@ def _to_bytes_or_none(text): class MailSender: def __init__( self, - smtphost="localhost", - mailfrom="scrapy@localhost", - smtpuser=None, - smtppass=None, - smtpport=25, - smtptls=False, - smtpssl=False, - debug=False, + smtphost: str = "localhost", + mailfrom: str = "scrapy@localhost", + smtpuser: Optional[str] = None, + smtppass: Optional[str] = None, + smtpport: int = 25, + smtptls: bool = False, + smtpssl: bool = False, + debug: bool = False, ): - self.smtphost = smtphost - self.smtpport = smtpport - self.smtpuser = _to_bytes_or_none(smtpuser) - self.smtppass = _to_bytes_or_none(smtppass) - self.smtptls = smtptls - self.smtpssl = smtpssl - self.mailfrom = mailfrom - self.debug = debug + self.smtphost: str = smtphost + self.smtpport: int = smtpport + self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) + self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtptls: bool = smtptls + self.smtpssl: bool = smtpssl + self.mailfrom: str = mailfrom + self.debug: bool = debug @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], @@ -68,17 +93,18 @@ class MailSender: def send( self, - to, - subject, - body, - cc=None, - attachs=(), - mimetype="text/plain", - charset=None, - _callback=None, - ): + to: Union[str, List[str]], + subject: str, + body: str, + cc: Union[str, List[str], None] = None, + attachs: Sequence[Tuple[str, str, IO]] = (), + mimetype: str = "text/plain", + charset: Optional[str] = None, + _callback: Optional[Callable[..., None]] = None, + ) -> Optional[Deferred]: from twisted.internet import reactor + msg: MIMEBase if attachs: msg = MIMEMultipart() else: @@ -125,19 +151,17 @@ class MailSender: "mailattachs": len(attachs), }, ) - return + return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) - dfd.addCallbacks( - callback=self._sent_ok, - errback=self._sent_failed, - callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)], - ) + dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) + dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd - def _sent_ok(self, result, to, cc, subject, nattachs): + def _sent_ok( + self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', @@ -149,7 +173,14 @@ class MailSender: }, ) - def _sent_failed(self, failure, to, cc, subject, nattachs): + def _sent_failed( + self, + failure: Failure, + to: List[str], + cc: List[str], + subject: str, + nattachs: int, + ) -> Failure: errstr = str(failure.value) logger.error( "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " @@ -165,13 +196,13 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs, msg): + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: from twisted.internet import reactor - msg = BytesIO(msg) - d = defer.Deferred() + msg_io = BytesIO(msg) + d: Deferred = Deferred() - factory = self._create_sender_factory(to_addrs, msg, d) + factory = self._create_sender_factory(to_addrs, msg_io, d) if self.smtpssl: reactor.connectSSL( @@ -182,10 +213,12 @@ class MailSender: return d - def _create_sender_factory(self, to_addrs, msg, d): + def _create_sender_factory( + self, to_addrs: List[str], msg: IO, d: Deferred + ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords = { + factory_keywords: Dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, @@ -202,7 +235,7 @@ class MailSender: to_addrs, msg, d, - **factory_keywords + **factory_keywords, ) factory.noisy = False return factory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 090588130..f60c726f9 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -23,7 +23,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -64,7 +64,10 @@ class MiddlewareManager: for clspath in mwlist: try: mwcls = load_object(clspath) - mw = create_instance(mwcls, settings, crawler) + if crawler is not None: + mw = build_from_crawler(mwcls, crawler) + else: + mw = build_from_settings(mwcls, settings) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..f9544d329 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ Item pipeline See documentation in docs/item-pipeline.rst """ + from typing import Any, List from twisted.internet.defer import Deferred diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5c09ab37e..47457f2a8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,12 +3,14 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + +from __future__ import annotations + import base64 import functools import hashlib import logging import mimetypes -import os import time from collections import defaultdict from contextlib import suppress @@ -16,7 +18,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import DefaultDict, Optional, Set, Union +from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -31,10 +33,13 @@ from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -42,6 +47,23 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string +def _md5sum(file: IO) -> str: + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. + + >>> from io import BytesIO + >>> _md5sum(BytesIO(b'file content to hash')) + '784406af91dd5a54fbb9c84c2236595a' + """ + m = hashlib.md5() # nosec + while True: + d = file.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + class FileException(Exception): """General media error exception""" @@ -66,11 +88,11 @@ class FSFilesStore: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime - except os.error: + except OSError: return {} with absolute_path.open("rb") as f: - checksum = md5sum(f) + checksum = _md5sum(f) return {"last_modified": last_modified, "checksum": checksum} @@ -299,7 +321,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() + m = hashlib.md5() # nosec ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -340,7 +362,9 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_URLS_FIELD = "file_urls" DEFAULT_FILES_RESULT_FIELD = "files" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured @@ -367,8 +391,8 @@ class FilesPipeline(MediaPipeline): super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -378,11 +402,15 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") @@ -431,7 +459,8 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addCallback(_onsuccess) + dfd.addErrback(lambda _: None) dfd.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", @@ -529,7 +558,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) - checksum = md5sum(buf) + checksum = _md5sum(buf) buf.seek(0) self.store.persist_file(path, buf, info) return checksum @@ -540,7 +569,7 @@ class FilesPipeline(MediaPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 1bd9832a8..e7ef06fb3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,25 +3,39 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + +from __future__ import annotations + import functools import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import Dict, Tuple +from os import PathLike +from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + FTPFilesStore, + GCSFilesStore, + S3FilesStore, + _md5sum, +) # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.misc import md5sum from scrapy.utils.python import get_func_args, to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class NoimagesDrop(DropItem): """Product with no images exception""" @@ -53,7 +67,9 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): try: from PIL import Image @@ -93,8 +109,8 @@ class ImagesPipeline(FilesPipeline): self._deprecated_convert_image = None @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -104,11 +120,15 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") @@ -124,7 +144,7 @@ class ImagesPipeline(FilesPipeline): for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) - checksum = md5sum(buf) + checksum = _md5sum(buf) width, height = image.size self.store.persist_file( path, @@ -224,9 +244,9 @@ class ImagesPipeline(FilesPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 75532034a..5f6c5cb07 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,6 +1,9 @@ +from __future__ import annotations + import functools import logging from collections import defaultdict +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -12,6 +15,11 @@ from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -67,9 +75,9 @@ class MediaPipeline: return formatted_key @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: try: - pipe = cls.from_settings(crawler.settings) + pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler @@ -98,10 +106,17 @@ class MediaPipeline: # Return cached result if request was already seen if fp in info.downloaded: - return defer_result(info.downloaded[fp]).addCallbacks(cb, eb) + d = defer_result(info.downloaded[fp]) + d.addCallback(cb) + if eb: + d.addErrback(eb) + return d # Otherwise, wait for result - wad = Deferred().addCallbacks(cb, eb) + wad = Deferred() + wad.addCallback(cb) + if eb: + wad.addErrback(eb) info.waiting[fp].append(wad) # Check if request is downloading right now to avoid doing it twice @@ -112,14 +127,14 @@ class MediaPipeline: info.downloading.add(fp) dfd = mustbe_deferred(self.media_to_download, request, info, item=item) dfd.addCallback(self._check_media_to_download, request, info, item=item) + dfd.addErrback(self._log_exception) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - dfd.addErrback( - lambda f: logger.error( - f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider} - ) - ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _log_exception(self, result): + logger.exception(result) + return result + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list @@ -132,23 +147,11 @@ class MediaPipeline: if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) + dfd.addCallback(self.media_downloaded, request, info, item=item) + dfd.addErrback(self.media_failed, request, info) return dfd def _cache_result_and_execute_waiters(self, result, fp, info): diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 62a9af477..58a47ef0f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,12 +1,33 @@ +from __future__ import annotations + import hashlib import logging +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + cast, +) -from scrapy.utils.misc import create_instance +from scrapy import Request +from scrapy.core.downloader import Downloader +from scrapy.utils.misc import build_from_crawler + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler logger = logging.getLogger(__name__) -def _path_safe(text): +def _path_safe(text: str) -> str: """ Return a filesystem-safe version of a string ``text`` @@ -20,10 +41,22 @@ def _path_safe(text): pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec return "-".join([pathable_slot, unique_slot]) +class QueueProtocol(Protocol): + """Protocol for downstream queues of ``ScrapyPriorityQueue``.""" + + def push(self, request: Request) -> None: ... + + def pop(self) -> Optional[Request]: ... + + def close(self) -> None: ... + + def __len__(self) -> int: ... + + class ScrapyPriorityQueue: """A priority queue implemented using multiple internal queues (typically, FIFO queues). It uses one internal queue for each priority value. The internal @@ -51,18 +84,30 @@ class ScrapyPriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, startprios=()): - self.crawler = crawler - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.queues = {} - self.curprio = None + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ): + self.crawler: Crawler = crawler + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.queues: Dict[int, QueueProtocol] = {} + self.curprio: Optional[int] = None self.init_prios(startprios) - def init_prios(self, startprios): + def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return @@ -71,18 +116,17 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) - def qfactory(self, key): - return create_instance( + def qfactory(self, key: int) -> QueueProtocol: + return build_from_crawler( self.downstream_queue_cls, - None, self.crawler, self.key + "/" + str(key), ) - def priority(self, request): + def priority(self, request: Request) -> int: return -request.priority - def push(self, request): + def push(self, request: Request) -> None: priority = self.priority(request) if priority not in self.queues: self.queues[priority] = self.qfactory(priority) @@ -91,9 +135,9 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self): + def pop(self) -> Optional[Request]: if self.curprio is None: - return + return None q = self.queues[self.curprio] m = q.pop() if not q: @@ -103,7 +147,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,30 +157,32 @@ class ScrapyPriorityQueue: if self.curprio is None: return None queue = self.queues[self.curprio] - return queue.peek() + # Protocols can't declare optional members + return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self): - active = [] + def close(self) -> List[int]: + active: List[int] = [] for p, q in self.queues.items(): active.append(p) q.close() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface: - def __init__(self, crawler): - self.downloader = crawler.engine.downloader + def __init__(self, crawler: Crawler): + assert crawler.engine + self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots): + def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] - def get_slot_key(self, request): - return self.downloader._get_slot_key(request, None) + def get_slot_key(self, request: Request) -> str: + return self.downloader.get_slot_key(request) - def _active_downloads(self, slot): + def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 @@ -150,10 +196,22 @@ class DownloaderAwarePriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Optional[Dict[str, Iterable[int]]] = None, + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' @@ -170,16 +228,18 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface = DownloaderInterface(crawler) - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.crawler = crawler + self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.crawler: Crawler = crawler - self.pqueues = {} # slot -> priority queue + self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def pqfactory(self, slot, startprios=()): + def pqfactory( + self, slot: str, startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, @@ -187,11 +247,11 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self): + def pop(self) -> Optional[Request]: stats = self._downloader_interface.stats(self.pqueues) if not stats: - return + return None slot = min(stats)[1] queue = self.pqueues[slot] @@ -200,14 +260,14 @@ class DownloaderAwarePriorityQueue: del self.pqueues[slot] return request - def push(self, request): + def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] queue.push(request) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -221,13 +281,13 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self): + def close(self) -> Dict[str, List[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): + def __contains__(self, slot: str) -> bool: return slot in self.pqueues diff --git a/scrapy/resolver.py b/scrapy/resolver.py index e2e8beff4..ba7cd716b 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,12 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer -from twisted.internet.base import ThreadedResolver +from twisted.internet.base import ReactorBase, ThreadedResolver +from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( + IAddress, IHostnameResolver, IHostResolution, IResolutionReceiver, @@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + # TODO: cache misses dnscache: LocalCache[str, Any] = LocalCache(10000) @@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver): Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size, timeout): + def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float): super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installResolver(self) - def getHostByName(self, name: str, timeout=None): + def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]: if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value - timeout = (self.timeout,) + # The timeout arg is typed as Sequence[int] but supports floats. + timeout = (self.timeout,) # type: ignore[assignment] d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d - def _cache_result(self, result, name): + def _cache_result(self, result: Any, name: str) -> Any: dnscache[name] = result return result @implementer(IHostResolution) class HostResolution: - def __init__(self, name): - self.name = name + def __init__(self, name: str): + self.name: str = name - def cancel(self): + def cancel(self) -> None: raise NotImplementedError() @provider(IResolutionReceiver) class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName): - self.resolutionReceiver = resolutionReceiver - self.hostName = hostName - self.addresses = [] + def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): + self.resolutionReceiver: IResolutionReceiver = resolutionReceiver + self.hostName: str = hostName + self.addresses: List[IAddress] = [] - def resolutionBegan(self, resolution): + def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) self.resolution = resolution - def addressResolved(self, address): + def addressResolved(self, address: IAddress) -> None: self.resolutionReceiver.addressResolved(address) self.addresses.append(address) - def resolutionComplete(self): + def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: dnscache[self.hostName] = self.addresses @@ -93,30 +104,30 @@ class CachingHostnameResolver: does not support setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size): - self.reactor = reactor - self.original_resolver = reactor.nameResolver + def __init__(self, reactor: ReactorBase, cache_size: int): + self.reactor: ReactorBase = reactor + self.original_resolver: IHostnameResolver = reactor.nameResolver dnscache.limit = cache_size @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installNameResolver(self) def resolveHostName( self, - resolutionReceiver, + resolutionReceiver: IResolutionReceiver, hostName: str, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): + portNumber: int = 0, + addressTypes: Optional[Sequence[Type[IAddress]]] = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: try: addresses = dnscache[hostName] except KeyError: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 9e411d4aa..702e50536 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,6 +2,7 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data @@ -21,9 +22,9 @@ class ResponseTypes: "application/xhtml+xml": "scrapy.http.HtmlResponse", "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse", "application/xml": "scrapy.http.XmlResponse", - "application/json": "scrapy.http.TextResponse", - "application/x-json": "scrapy.http.TextResponse", - "application/json-amazonui-streaming": "scrapy.http.TextResponse", + "application/json": "scrapy.http.JsonResponse", + "application/x-json": "scrapy.http.JsonResponse", + "application/json-amazonui-streaming": "scrapy.http.JsonResponse", "application/javascript": "scrapy.http.TextResponse", "application/x-javascript": "scrapy.http.TextResponse", "text/xml": "scrapy.http.XmlResponse", diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 6ea2bfd97..a33f73306 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,9 +3,10 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from warnings import warn +from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -18,12 +19,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): +def decode_robotstxt( + robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False +) -> str: try: if to_native_str_type: - robotstxt_body = to_unicode(robotstxt_body) + body_decoded = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8") + body_decoded = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): exc_info=sys.exc_info(), extra={"spider": spider}, ) - robotstxt_body = "" - return robotstxt_body + body_decoded = "" + return body_decoded class RobotParser(metaclass=ABCMeta): @@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from urllib.robotparser import RobotFileParser - self.spider = spider - robotstxt_body = decode_robotstxt( - robotstxt_body, spider, to_native_str_type=True - ) - self.rp = RobotFileParser() - self.rp.parse(robotstxt_body.splitlines()) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + self.rp: RobotFileParser = RobotFileParser() + self.rp.parse(body_decoded.splitlines()) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider = spider + self.spider: Optional[Spider] = spider self.rp = Robots.parse("", robotstxt_body) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider = spider - self.rp = RobotExclusionRulesParser() - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from protego import Protego - self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp = Protego.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..e852aadc7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,12 +1,14 @@ """ XPath selectors based on lxml """ + from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes +from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] @@ -45,7 +47,7 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` or ``None`` (default). If ``type`` is ``None``, the selector automatically chooses the best type @@ -87,7 +89,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault("base_url", response.url) + kwargs.setdefault("base_url", get_base_url(response)) self.response = response diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d270a72f4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..932475fb5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -101,6 +101,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -177,7 +178,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" +FTP_PASSWORD = "guest" # nosec FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None @@ -239,7 +240,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = [] +METAREFRESH_IGNORE_TAGS = ["noscript"] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" @@ -260,7 +261,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests @@ -301,7 +302,6 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..2c22d3d8f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,8 +3,10 @@ See documentation in docs/topics/shell.rst """ + import os import signal +from typing import Any, Callable, Dict, Optional, Tuple, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -25,18 +27,32 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) + relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) - def __init__(self, crawler, update_vars=None, code=None): - self.crawler = crawler - self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider = None - self.inthread = not threadable.isInIOThread() - self.code = code - self.vars = {} + def __init__( + self, + crawler: Crawler, + update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + code: Optional[str] = None, + ): + self.crawler: Crawler = crawler + self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + lambda x: None + ) + self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) + self.spider: Optional[Spider] = None + self.inthread: bool = not threadable.isInIOThread() + self.code: Optional[str] = code + self.vars: Dict[str, Any] = {} - def start(self, url=None, request=None, response=None, spider=None, redirect=True): + def start( + self, + url: Optional[str] = None, + request: Optional[Request] = None, + response: Optional[Response] = None, + spider: Optional[Spider] = None, + redirect: bool = True, + ) -> None: # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: @@ -49,7 +65,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # nosec else: """ Detect interactive shell setting in scrapy.cfg @@ -76,7 +92,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request, spider): + def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -84,10 +100,11 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) + assert self.crawler.engine self.crawler.engine.crawl(request) return d - def _open_spider(self, request, spider): + def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: if self.spider: return self.spider @@ -95,11 +112,18 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider + assert self.crawler.engine self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider - def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): + def fetch( + self, + request_or_url: Union[Request, str], + spider: Optional[Spider] = None, + redirect: bool = True, + **kwargs: Any, + ) -> None: from twisted.internet import reactor if isinstance(request_or_url, Request): @@ -122,7 +146,12 @@ class Shell: pass self.populate_vars(response, request, spider) - def populate_vars(self, response=None, request=None, spider=None): + def populate_vars( + self, + response: Optional[Response] = None, + request: Optional[Request] = None, + spider: Optional[Spider] = None, + ) -> None: import scrapy self.vars["scrapy"] = scrapy @@ -140,10 +169,10 @@ class Shell: if not self.code: self.vars["banner"] = self.get_help() - def print_help(self): + def print_help(self) -> None: print(self.get_help()) - def get_help(self): + def get_help(self) -> str: b = [] b.append("Available Scrapy objects:") b.append( @@ -167,11 +196,11 @@ class Shell: return "\n".join(f"[s] {line}" for line in b) - def _is_relevant(self, value): + def _is_relevant(self, value: Any) -> bool: return isinstance(value, self.relevant_classes) or is_item(value) -def inspect_response(response, spider): +def inspect_response(response: Response, spider: Spider) -> None: """Open a shell to inspect the given response""" # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed @@ -180,7 +209,7 @@ def inspect_response(response, spider): signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request): +def _request_deferred(request: Request) -> defer.Deferred: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -194,15 +223,17 @@ def _request_deferred(request): request_callback = request.callback request_errback = request.errback - def _restore_callbacks(result): + def _restore_callbacks(result: Any) -> Any: request.callback = request_callback request.errback = request_errback return result - d = defer.Deferred() + d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: - d.addCallbacks(request.callback, request.errback) + d.addCallback(request.callback) + if request.errback: + d.addErrback(request.errback) request.callback, request.errback = d.callback, d.errback return d diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..35c869a75 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,6 +3,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a5214702d..50c93ac9f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,6 +3,7 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging @@ -12,15 +13,21 @@ from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +warnings.warn( + "The scrapy.spidermiddlewares.offsite module is deprecated, use " + "scrapy.downloadermiddlewares.offsite instead.", + ScrapyDeprecationWarning, +) + if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..a0b6851e5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..bef041325 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging @@ -16,11 +17,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Callable + + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, Self from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + from scrapy.utils.log import SpiderLoggerAdapter + + CallbackT = Callable[Concatenate[Response, ...], Any] class Spider(object_ref): @@ -33,7 +40,7 @@ class Spider(object_ref): def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: - self.name = name + self.name: str = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) @@ -41,9 +48,11 @@ class Spider(object_ref): self.start_urls: List[str] = [] @property - def logger(self) -> logging.LoggerAdapter: + def logger(self) -> SpiderLoggerAdapter: + from scrapy.utils.log import SpiderLoggerAdapter + logger = logging.getLogger(self.name) - return logging.LoggerAdapter(logger, {"spider": self}) + return SpiderLoggerAdapter(logger, {"spider": self}) def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level @@ -61,8 +70,8 @@ class Spider(object_ref): return spider def _set_crawler(self, crawler: Crawler) -> None: - self.crawler = crawler - self.settings = crawler.settings + self.crawler: Crawler = crawler + self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self) -> Iterable[Request]: @@ -78,10 +87,14 @@ class Spider(object_ref): def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response: Response, **kwargs: Any) -> Any: - raise NotImplementedError( - f"{self.__class__.__name__}.parse callback is not defined" - ) + if TYPE_CHECKING: + parse: CallbackT + else: + + def parse(self, response: Response, **kwargs: Any) -> Any: + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) @classmethod def update_settings(cls, settings: BaseSettings) -> None: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 31e845716..48c830d2a 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -5,29 +5,65 @@ for scraping typical web sites that requires crawling pages. See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import copy -from typing import AsyncIterable, Awaitable, Sequence +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Awaitable, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + TypeVar, + Union, + cast, +) + +from twisted.python.failure import Failure from scrapy.http import HtmlResponse, Request, Response +from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _identity(x): + from scrapy.crawler import Crawler + + +_T = TypeVar("_T") +ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessRequestT = Callable[[Request, Response], Optional[Request]] + + +def _identity(x: _T) -> _T: return x -def _identity_process_request(request, response): +def _identity_process_request( + request: Request, response: Response +) -> Optional[Request]: return request -def _get_method(method, spider): +def _get_method( + method: Union[Callable, str, None], spider: Spider +) -> Optional[Callable]: if callable(method): return method if isinstance(method, str): return getattr(spider, method, None) + return None _default_link_extractor = LinkExtractor() @@ -36,37 +72,46 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor=None, - callback=None, - cb_kwargs=None, - follow=None, - process_links=None, - process_request=None, - errback=None, + link_extractor: Optional[LinkExtractor] = None, + callback: Union[Callable, str, None] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + follow: Optional[bool] = None, + process_links: Union[ProcessLinksT, str, None] = None, + process_request: Union[ProcessRequestT, str, None] = None, + errback: Union[Callable[[Failure], Any], str, None] = None, ): - self.link_extractor = link_extractor or _default_link_extractor - self.callback = callback - self.errback = errback - self.cb_kwargs = cb_kwargs or {} - self.process_links = process_links or _identity - self.process_request = process_request or _identity_process_request - self.follow = follow if follow is not None else not callback + self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor + self.callback: Union[Callable, str, None] = callback + self.errback: Union[Callable[[Failure], Any], str, None] = errback + self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.process_links: Union[ProcessLinksT, str] = process_links or _identity + self.process_request: Union[ProcessRequestT, str] = ( + process_request or _identity_process_request + ) + self.follow: bool = follow if follow is not None else not callback - def _compile(self, spider): + def _compile(self, spider: Spider) -> None: + # this replaces method names with methods and we can't express this in type hints self.callback = _get_method(self.callback, spider) - self.errback = _get_method(self.errback, spider) - self.process_links = _get_method(self.process_links, spider) - self.process_request = _get_method(self.process_request, spider) + self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) + self.process_links = cast( + ProcessLinksT, _get_method(self.process_links, spider) + ) + self.process_request = cast( + ProcessRequestT, _get_method(self.process_request, spider) + ) class CrawlSpider(Spider): rules: Sequence[Rule] = () + _rules: List[Rule] + _follow_links: bool - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self._parse_response( response=response, callback=self.parse_start_url, @@ -74,46 +119,57 @@ class CrawlSpider(Spider): follow=True, ) - def parse_start_url(self, response, **kwargs): + def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: list): + def process_results(self, response: Response, results: Any) -> Any: return results - def _build_request(self, rule_index, link): + def _build_request(self, rule_index: int, link: Link) -> Request: return Request( url=link.url, callback=self._callback, errback=self._errback, - meta=dict(rule=rule_index, link_text=link.text), + meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response): + def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen = set() + seen: Set[Link] = set() for rule_index, rule in enumerate(self._rules): - links = [ + links: List[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen ] - for link in rule.process_links(links): + for link in cast(ProcessLinksT, rule.process_links)(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule.process_request(request, response) + yield cast(ProcessRequestT, rule.process_request)(request, response) - def _callback(self, response, **cb_kwargs): - rule = self._rules[response.meta["rule"]] + def _callback(self, response: Response, **cb_kwargs: Any) -> Any: + rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( - response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow + response, + cast(Callable, rule.callback), + {**rule.cb_kwargs, **cb_kwargs}, + rule.follow, ) - def _errback(self, failure): - rule = self._rules[failure.request.meta["rule"]] - return self._handle_failure(failure, rule.errback) + def _errback(self, failure: Failure) -> Iterable[Any]: + rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined] + return self._handle_failure( + failure, cast(Callable[[Failure], Any], rule.errback) + ) - async def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response( + self, + response: Response, + callback: Optional[Callable], + cb_kwargs: Dict[str, Any], + follow: bool = True, + ) -> AsyncIterable[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () if isinstance(cb_res, AsyncIterable): @@ -128,20 +184,21 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item - def _handle_failure(self, failure, errback): + def _handle_failure( + self, failure: Failure, errback: Optional[Callable[[Failure], Any]] + ) -> Iterable[Any]: if errback: results = errback(failure) or () - for request_or_item in iterate_spider_output(results): - yield request_or_item + yield from iterate_spider_output(results) - def _compile_rules(self): + def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: self._rules.append(copy.copy(rule)) self._rules[-1]._compile(self) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool( "CRAWLSPIDER_FOLLOW_LINKS", True diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..9dd8a5d68 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,10 +4,14 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider -from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output @@ -21,11 +25,13 @@ class XMLFeedSpider(Spider): use iternodes, since it's a faster and cleaner. """ - iterator = "iternodes" - itertag = "item" - namespaces = () + iterator: str = "iternodes" + itertag: str = "item" + namespaces: Sequence[Tuple[str, str]] = () - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This overridable method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, @@ -35,20 +41,20 @@ class XMLFeedSpider(Spider): """ return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """You can override this function in order to make any changes you want to into the feed before parsing it. This function must return a response. """ return response - def parse_node(self, response, selector): + def parse_node(self, response: Response, selector: Selector) -> Any: """This method must be overridden with your custom spider functionality""" if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError - def parse_nodes(self, response, nodes): + def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. @@ -58,23 +64,27 @@ class XMLFeedSpider(Spider): for selector in nodes: ret = iterate_spider_output(self.parse_node(response, selector)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_node"): raise NotConfigured( "You must define parse_node method in order to scrape this XML feed" ) response = self.adapt_response(response) + nodes: Iterable[Selector] if self.iterator == "iternodes": nodes = self._iternodes(response) elif self.iterator == "xml": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="xml") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") elif self.iterator == "html": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="html") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") @@ -83,12 +93,12 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) - def _iternodes(self, response): - for node in xmliter(response, self.itertag): + def _iternodes(self, response: Response) -> Iterable[Selector]: + for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node - def _register_namespaces(self, selector): + def _register_namespaces(self, selector: Selector) -> None: for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) @@ -102,27 +112,29 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = ( + delimiter: Optional[str] = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar = ( + quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers = None + headers: Optional[List[str]] = None - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This method has the same purpose as the one in XMLFeedSpider""" return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response, row): + def parse_row(self, response: Response, row: Dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError - def parse_rows(self, response): + def parse_rows(self, response: Response) -> Any: """Receives a response and a dict (representing each row) with a key for each provided (or detected) header of the CSV file. This spider also gives the opportunity to override adapt_response and @@ -133,10 +145,9 @@ class CSVFeedSpider(Spider): response, self.delimiter, self.headers, quotechar=self.quotechar ): ret = iterate_spider_output(self.parse_row(response, row)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_row"): raise NotConfigured( "You must define parse_row method in order to scrape this CSV feed" diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 3cb215b0f..a0898a0cf 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,3 +1,7 @@ +from typing import Any, Iterable, Optional, cast + +from scrapy import Request +from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output @@ -5,17 +9,17 @@ from scrapy.utils.spider import iterate_spider_output class InitSpider(Spider): """Base Spider with initialization facilities""" - def start_requests(self): - self._postinit_reqs = super().start_requests() - return iterate_spider_output(self.init_request()) + def start_requests(self) -> Iterable[Request]: + self._postinit_reqs: Iterable[Request] = super().start_requests() + return cast(Iterable[Request], iterate_spider_output(self.init_request())) - def initialized(self, response=None): + def initialized(self, response: Optional[Response] = None) -> Any: """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ return self.__dict__.pop("_postinit_reqs") - def init_request(self): + def init_request(self) -> Any: """This function should return one initialization request, with the self.initialized method as callback. When the self.initialized method is called this spider is considered initialized. If you need to perform diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index aaf75a519..d082fbfdb 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,42 +1,80 @@ +from __future__ import annotations + import logging import re +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) -from scrapy.http import Request, XmlResponse +from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () - sitemap_rules = [("", "parse")] - sitemap_follow = [""] - sitemap_alternate_links = False + sitemap_urls: Sequence[str] = () + sitemap_rules: Sequence[ + Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + ] = [("", "parse")] + sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] + sitemap_alternate_links: bool = False + _max_size: int + _warn_size: int - def __init__(self, *a, **kw): + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: + spider = super().from_crawler(crawler, *args, **kwargs) + spider._max_size = getattr( + spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") + ) + spider._warn_size = getattr( + spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE") + ) + return spider + + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs = [] + self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = getattr(self, c) + c = cast(Callable, getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow = [regex(x) for x in self.sitemap_follow] + self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - def start_requests(self): + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, entries): + def sitemap_filter( + self, entries: Iterable[Dict[str, Any]] + ) -> Iterable[Dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for entry in entries: - yield entry + yield from entries - def _parse_sitemap(self, response): + def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) @@ -64,14 +102,26 @@ class SitemapSpider(Spider): yield Request(loc, callback=c) break - def _get_sitemap_body(self, response): + def _get_sitemap_body(self, response: Response) -> Optional[bytes]: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - return gunzip(response.body) + uncompressed_size = len(response.body) + max_size = response.meta.get("download_maxsize", self._max_size) + warn_size = response.meta.get("download_warnsize", self._warn_size) + try: + body = gunzip(response.body, max_size=max_size) + except _DecompressionMaxSizeExceeded: + return None + if uncompressed_size < warn_size <= len(body): + logger.warning( + f"{response} body size after decompression ({len(body)} B) " + f"is larger than the download warning size ({warn_size} B)." + ) + return body # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, @@ -83,15 +133,16 @@ class SitemapSpider(Spider): # in other word, here, we have plain XML if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): return response.body + return None -def regex(x): +def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x -def iterloc(it, alt=False): +def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..6f80ee388 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -2,20 +2,28 @@ Scheduler queues """ +from __future__ import annotations + import marshal -import pickle +import pickle # nosec from os import PathLike from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue +from scrapy import Request +from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _with_mkdir(queue_class): - class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): + +def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: + class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] + def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -24,18 +32,23 @@ def _with_mkdir(queue_class): return DirectoriesCreated -def _serializable_queue(queue_class, serialize, deserialize): - class SerializableQueue(queue_class): - def push(self, obj): +def _serializable_queue( + queue_class: Type[queue.BaseQueue], + serialize: Callable[[Any], bytes], + deserialize: Callable[[bytes], Any], +) -> Type[queue.BaseQueue]: + class SerializableQueue(queue_class): # type: ignore[valid-type,misc] + def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) - def pop(self): + def pop(self) -> Optional[Any]: s = super().pop() if s: return deserialize(s) + return None - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize): ) from ex if s: return deserialize(s) + return None return SerializableQueue -def _scrapy_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): +def _scrapy_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] + def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @classmethod - def from_crawler(cls, crawler, key, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any + ) -> Self: return cls(crawler, key) - def push(self, request): - request = request.to_dict(spider=self.spider) - return super().push(request) + def push(self, request: Request) -> None: + request_dict = request.to_dict(spider=self.spider) + super().push(request_dict) - def pop(self): + def pop(self) -> Optional[Request]: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class): return ScrapyRequestQueue -def _scrapy_non_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): +def _scrapy_non_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class): return ScrapyRequestQueue -def _pickle_serialize(obj): +def _pickle_serialize(obj: Any) -> bytes: try: return pickle.dumps(obj, protocol=4) # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) @@ -122,17 +142,18 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e +# queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) # public queue classes @@ -140,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) -FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) -LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..ab571a3ab 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,6 +1,7 @@ """ Scrapy extension for collecting scraping stats """ + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index ecb1e5e5c..b4779e555 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py new file mode 100644 index 000000000..591737b8e --- /dev/null +++ b/scrapy/utils/_compression.py @@ -0,0 +1,125 @@ +import zlib +from io import BytesIO +from warnings import warn + +from scrapy.exceptions import ScrapyDeprecationWarning + +try: + try: + import brotli + except ImportError: + import brotlicffi as brotli +except ImportError: + pass +else: + try: + brotli.Decompressor.process + except AttributeError: + warn( + ( + "You have brotlipy installed, and Scrapy will use it, but " + "Scrapy support for brotlipy is deprecated and will stop " + "working in a future version of Scrapy. brotlipy itself is " + "deprecated, it has been superseded by brotlicffi. Please, " + "uninstall brotlipy and install brotli or brotlicffi instead. " + "brotlipy has the same import name as brotli, so keeping both " + "installed is strongly discouraged." + ), + ScrapyDeprecationWarning, + ) + + def _brotli_decompress(decompressor, data): + return decompressor.decompress(data) + + else: + + def _brotli_decompress(decompressor, data): + return decompressor.process(data) + + +try: + import zstandard +except ImportError: + pass + + +_CHUNK_SIZE = 65536 # 64 KiB + + +class _DecompressionMaxSizeExceeded(ValueError): + pass + + +def _inflate(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zlib.decompressobj() + raw_decompressor = zlib.decompressobj(wbits=-15) + input_stream = BytesIO(data) + output_stream = BytesIO() + output_chunk = b"." + decompressed_size = 0 + while output_chunk: + input_chunk = input_stream.read(_CHUNK_SIZE) + try: + output_chunk = decompressor.decompress(input_chunk) + except zlib.error: + if decompressor != raw_decompressor: + # ugly hack to work with raw deflate content that may + # be sent by microsoft servers. For more information, see: + # http://carsten.codimi.de/gzip.yaws/ + # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx + # http://www.gzip.org/zlib/zlib_faq.html#faq38 + decompressor = raw_decompressor + output_chunk = decompressor.decompress(input_chunk) + else: + raise + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." + ) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() + + +def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = brotli.Decompressor() + input_stream = BytesIO(data) + output_stream = BytesIO() + output_chunk = b"." + decompressed_size = 0 + while output_chunk: + input_chunk = input_stream.read(_CHUNK_SIZE) + output_chunk = _brotli_decompress(decompressor, input_chunk) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." + ) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() + + +def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zstandard.ZstdDecompressor() + stream_reader = decompressor.stream_reader(BytesIO(data)) + output_stream = BytesIO() + output_chunk = b"." + decompressed_size = 0 + while output_chunk: + output_chunk = stream_reader.read(_CHUNK_SIZE) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." + ) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..f6f704d4b 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -14,7 +14,7 @@ class Root(Resource): def render(self, request): total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") args = request.args.copy() for nl in nlist: diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 100f040bb..bf1803115 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,17 +1,27 @@ from functools import wraps +from typing import Any, Callable, Dict, Iterable, Optional + +EmbedFuncT = Callable[..., None] +KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] -def _embed_ipython_shell(namespace={}, banner=""): +def _embed_ipython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start an IPython Shell""" try: from IPython.terminal.embed import InteractiveShellEmbed from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + InteractiveShellEmbed, + ) + from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] + load_default_config, + ) @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""): return wrapper -def _embed_bpython_shell(namespace={}, banner=""): +def _embed_bpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper -def _embed_ptpython_shell(namespace={}, banner=""): +def _embed_ptpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) return wrapper -def _embed_standard_shell(namespace={}, banner=""): +def _embed_standard_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a standard python shell""" import code @@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""): readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper -DEFAULT_PYTHON_SHELLS = { +DEFAULT_PYTHON_SHELLS: KnownShellsT = { "ptpython": _embed_ptpython_shell, "ipython": _embed_ipython_shell, "bpython": _embed_bpython_shell, @@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = { } -def get_shell_embed_func(shells=None, known_shells=None): +def get_shell_embed_func( + shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None +) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None): continue -def start_python_console(namespace=None, banner="", shells=None): +def start_python_console( + namespace: Optional[Dict[str, Any]] = None, + banner: str = "", + shells: Optional[Iterable[str]] = None, +) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 04186559f..7e82dd519 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,21 +1,34 @@ +from __future__ import annotations + import warnings from functools import wraps -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable, TypeVar from twisted.internet import defer, threads from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec -def deprecated(use_instead: Any = None) -> Callable: + _P = ParamSpec("_P") + + +_T = TypeVar("_T") + + +def deprecated( + use_instead: Any = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func: Callable) -> Callable: + def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) - def wrapped(*args: Any, **kwargs: Any) -> Any: + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -30,23 +43,23 @@ def deprecated(use_instead: Any = None) -> Callable: return deco -def defers(func: Callable) -> Callable[..., Deferred]: +def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func: Callable) -> Callable[..., Deferred]: +def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..abb7e1726 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,21 +1,23 @@ """ Helper functions for dealing with Twisted deferreds """ + +from __future__ import annotations + import asyncio import inspect from asyncio import Future from functools import wraps from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Dict, - Generator, Iterable, Iterator, List, @@ -36,6 +38,14 @@ from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until @@ -73,7 +83,31 @@ def defer_result(result: Any) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: +@overload +def mustbe_deferred( + f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, Coroutine[Deferred[Any], Any, _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +def mustbe_deferred( + f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -91,7 +125,11 @@ def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any + iterable: Iterable[_T], + count: int, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -103,7 +141,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator): +class _AsyncCooperatorAdapter(Iterator[Deferred]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -151,28 +189,30 @@ class _AsyncCooperatorAdapter(Iterator): def __init__( self, - aiterable: AsyncIterable, - callable: Callable, - *callable_args: Any, - **callable_kwargs: Any, + aiterable: AsyncIterable[_T], + callable: Callable[Concatenate[_T, _P], Any], + *callable_args: _P.args, + **callable_kwargs: _P.kwargs, ): - self.aiterator: AsyncIterator = aiterable.__aiter__() - self.callable: Callable = callable + self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() + self.callable: Callable[Concatenate[_T, _P], Any] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] - self.anext_deferred: Optional[Deferred] = None + self.anext_deferred: Optional[Deferred[_T]] = None - def _callback(self, result: Any) -> None: + def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None - result = self.callable(result, *self.callable_args, **self.callable_kwargs) + callable_result = self.callable( + result, *self.callable_args, **self.callable_kwargs + ) d = self.waiting_deferreds.pop(0) - if isinstance(result, Deferred): - result.chainDeferred(d) + if isinstance(callable_result, Deferred): + callable_result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: @@ -206,11 +246,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, + async_iterable: AsyncIterable[_T], count: int, - callable: Callable, - *args: Any, - **named: Any, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -220,7 +260,10 @@ def parallel_async( def process_chain( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() @@ -231,23 +274,17 @@ def process_chain( def process_chain_both( - callbacks: Iterable[Callable], - errbacks: Iterable[Callable], + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], input: Any, - *a: Any, - **kw: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks( - callback=cb, - errback=eb, - callbackArgs=a, - callbackKeywords=kw, - errbackArgs=a, - errbackKeywords=kw, - ) + d.addCallback(cb, *a, **kw) + d.addErrback(eb, *a, **kw) if isinstance(input, failure.Failure): d.errback(input) else: @@ -256,20 +293,27 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) + d.addCallback(lambda r: [x[1] for x in r]) + d.addErrback(lambda f: f.value.subFailure) return d def iter_errback( - iterable: Iterable, errback: Callable, *a: Any, **kw: Any -) -> Generator: + iterable: Iterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> Iterable[_T]: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -284,8 +328,11 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any -) -> AsyncGenerator: + aiterable: AsyncIterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> AsyncIterable[_T]: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -300,17 +347,14 @@ async def aiter_errback( _CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) -_T = TypeVar("_T") @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _CT) -> Deferred: ... @overload -def deferred_from_coro(o: _T) -> _T: - ... +def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Union[Deferred, _T]: @@ -328,7 +372,9 @@ def deferred_from_coro(o: _T) -> Union[Deferred, _T]: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: +def deferred_f_from_coro_f( + coro_f: Callable[_P, Coroutine[Any, Any, _T]] +) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -336,13 +382,15 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args: Any, **coro_kwargs: Any) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: +def maybeDeferred_coro( + f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs +) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..e0f2ac763 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..fdcf484d4 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,14 +1,15 @@ """Some debugging functions for working with the Scrapy engine""" +from __future__ import annotations + # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import Any, List, Tuple -if TYPE_CHECKING: - from scrapy.core.engine import ExecutionEngine +from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -30,14 +31,14 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: checks: List[Tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # nosec except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] return checks -def format_engine_status(engine: "ExecutionEngine") -> str: +def format_engine_status(engine: ExecutionEngine) -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str: return s -def print_engine_status(engine: "ExecutionEngine") -> None: +def print_engine_status(engine: ExecutionEngine) -> None: print(format_engine_status(engine)) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c7f74030e..2e487d88b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,31 +1,41 @@ import struct from gzip import GzipFile from io import BytesIO -from typing import List from scrapy.http import Response +from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded -def gunzip(data: bytes) -> bytes: + +def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() chunk = b"." + decompressed_size = 0 while chunk: try: - chunk = f.read1(8196) - output_list.append(chunk) + chunk = f.read1(_CHUNK_SIZE) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty - if output_list: + # some pages are quite small so output_stream is empty + if output_stream.getbuffer().nbytes > 0: break raise - return b"".join(output_list) + decompressed_size += len(chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." + ) + output_stream.write(chunk) + output_stream.seek(0) + return output_stream.read() def gzip_magic_number(response: Response) -> bool: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 55362efdf..cd6e9d04e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,7 +3,6 @@ import logging import re from io import StringIO from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -16,14 +15,15 @@ from typing import ( cast, overload, ) +from warnings import warn +from lxml import etree # nosec + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode -if TYPE_CHECKING: - from lxml._types import SupportsReadClose - logger = logging.getLogger(__name__) @@ -38,6 +38,16 @@ def xmliter( - a unicode string - a string encoded as utf-8 """ + warn( + ( + "xmliter is deprecated and its use strongly discouraged because " + "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See " + "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9" + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + nodename_patt = re.escape(nodename) DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) @@ -81,15 +91,35 @@ def xmliter_lxml( namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - from lxml import etree - reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + reader, + encoding=reader.encoding, + events=("end", "start-ns"), + resolve_entities=False, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) - for _, node in iterable: + needs_namespace_resolution = not namespace and ":" in nodename + if needs_namespace_resolution: + prefix, nodename = nodename.split(":", maxsplit=1) + for event, data in iterable: + if event == "start-ns": + assert isinstance(data, tuple) + if needs_namespace_resolution: + _prefix, _namespace = data + if _prefix != prefix: + continue + namespace = _namespace + needs_namespace_resolution = False + selxpath = f"//{prefix}:{nodename}" + tag = f"{{{namespace}}}{nodename}" + continue + assert isinstance(data, etree._Element) + node = data + if node.tag != tag: + continue nodetext = etree.tostring(node, encoding="unicode") node.clear() xs = Selector(text=nodetext, type="xml") @@ -192,18 +222,17 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str( + obj: Union[Response, str, bytes], unicode: Literal[False] +) -> bytes: ... def _body_or_str( diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2a38f151a..430a91e95 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -4,7 +4,17 @@ import logging import sys from logging.config import dictConfig from types import TracebackType -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + List, + MutableMapping, + Optional, + Tuple, + Type, + Union, + cast, +) from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -238,3 +248,16 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: args = logkws if not logkws.get("args") else logkws["args"] return (level, message, args) + + +class SpiderLoggerAdapter(logging.LoggerAdapter): + def process( + self, msg: str, kwargs: MutableMapping[str, Any] + ) -> Tuple[str, MutableMapping[str, Any]]: + """Method that augments logging with additional 'extra' data""" + if isinstance(kwargs.get("extra"), MutableMapping): + kwargs["extra"].update(self.extra) + else: + kwargs["extra"] = self.extra + + return msg, kwargs diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a9364bea2..faf52e44a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,7 @@ """Helper functions which don't fit anywhere else""" + +from __future__ import annotations + import ast import hashlib import inspect @@ -21,18 +24,23 @@ from typing import ( Iterable, List, Optional, + Type, + TypeVar, Union, cast, ) +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: from scrapy import Spider - + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes +T = TypeVar("T") def arg_to_iter(arg: Any) -> Iterable[Any]: @@ -111,7 +119,15 @@ def md5sum(file: IO) -> str: >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() + warnings.warn( + ( + "The scrapy.utils.misc.md5sum function is deprecated, and will be " + "removed in a future version of Scrapy." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + m = hashlib.md5() # nosec while True: d = file.read(8096) if not d: @@ -142,6 +158,13 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an extension has not been implemented correctly). """ + warnings.warn( + "The create_instance() function is deprecated. " + "Please use build_from_crawler() or build_from_settings() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if settings is None: if crawler is None: raise ValueError("Specify at least one of settings and crawler.") @@ -160,6 +183,49 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return instance +def build_from_crawler( + objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any +) -> T: + """Construct a class instance using its ``from_crawler`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_crawler"): + instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] + method_name = "from_crawler" + elif hasattr(objcls, "from_settings"): + instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined] + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return cast(T, instance) + + +def build_from_settings( + objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any +) -> T: + """Construct a class instance using its ``from_settings`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_settings"): + instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined] + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return cast(T, instance) + + @contextmanager def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and @@ -234,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: +def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..5985a847e 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -24,7 +24,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0b5dc324f..578cde2ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,9 @@ """ This module contains essential stuff that should've come with Python itself ;) """ + +from __future__ import annotations + import collections.abc import gc import inspect @@ -10,6 +13,7 @@ import weakref from functools import partial, wraps from itertools import chain from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, @@ -24,12 +28,21 @@ from typing import ( Optional, Pattern, Tuple, + TypeVar, Union, overload, ) from scrapy.utils.asyncgen import as_async_generator +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def flatten(x: Iterable) -> list: """flatten(sequence) -> list @@ -57,8 +70,7 @@ def iflatten(x: Iterable) -> Iterable: Similar to ``.flatten()``, but returns iterator instead""" for el in x: if is_listlike(el): - for el_ in iflatten(el): - yield el_ + yield from iflatten(el) else: yield el @@ -162,21 +174,26 @@ def re_rsearch( pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): - matches = [match for match in pattern.finditer(chunk)] + matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() return offset + start, offset + end return None -def memoizemethod_noargs(method: Callable) -> Callable: +_SelfT = TypeVar("_SelfT") + + +def memoizemethod_noargs( + method: Callable[Concatenate[_SelfT, _P], _T] +) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: + def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -286,13 +303,11 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Mapping) -> dict: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: - ... +def without_none_values(iterable: Iterable) -> Iterable: ... def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: @@ -316,7 +331,7 @@ def global_object_name(obj: Any) -> str: >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return f"{obj.__module__}.{obj.__name__}" + return f"{obj.__module__}.{obj.__qualname__}" if hasattr(sys, "pypy_version_info"): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ad3d1d8bc..5af6d22eb 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,18 +1,41 @@ +from __future__ import annotations + import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence, Type +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + List, + Optional, + Tuple, + Type, + TypeVar, +) from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall +from twisted.internet.protocol import ServerFactory +from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec -def listen_tcp(portrange, host, factory): + _P = ParamSpec("_P") + +_T = TypeVar("_T") + + +def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -20,8 +43,6 @@ def listen_tcp(portrange, host, factory): raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, "__iter__"): - return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1] + 1): @@ -32,14 +53,14 @@ def listen_tcp(portrange, host, factory): raise -class CallLaterOnce: +class CallLaterOnce(Generic[_T]): """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func: Callable, *a: Any, **kw: Any): - self._func: Callable = func - self._a: Sequence[Any] = a + def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): + self._func: Callable[_P, _T] = func + self._a: Tuple[Any, ...] = a self._kw: Dict[str, Any] = kw self._call: Optional[DelayedCall] = None @@ -53,7 +74,7 @@ class CallLaterOnce: if self._call: self._call.cancel() - def __call__(self) -> Any: + def __call__(self) -> _T: self._call = None return self._func(*self._a, **self._kw) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 24fcbd85e..c86f9fe39 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,6 +3,8 @@ This module provides some useful functions for working with scrapy.http.Request objects """ +from __future__ import annotations + import hashlib import json import warnings @@ -32,10 +34,10 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - from scrapy.crawler import Crawler + # typing.Self requires Python 3.11 + from typing_extensions import Self -_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" -_deprecated_fingerprint_cache = WeakKeyDictionary() + from scrapy.crawler import Crawler def _serialize_headers( @@ -44,125 +46,12 @@ def _serialize_headers( for header in headers: if header in request.headers: yield header - for value in request.headers.getlist(header): - yield value + yield from request.headers.getlist(header) -def request_fingerprint( - request: Request, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, - keep_fragments: bool = False, -) -> str: - """ - Return the request fingerprint as an hexadecimal string. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - if include_headers or keep_fragments: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component because you " - "need a non-default fingerprinting algorithm, and you are OK " - "with that non-default fingerprinting algorithm being used by " - "all Scrapy components and not just the one calling this " - "function, use crawler.request_fingerprinter.fingerprint() " - "instead in your Scrapy component (you can get the crawler " - "object from the 'from_crawler' class method), and use the " - "'REQUEST_FINGERPRINTER_CLASS' setting to configure your " - "non-default fingerprinting algorithm.\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "If you switch to 'fingerprint()', or assign the " - "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses " - "'fingerprint()', the generated fingerprints will not only be " - "bytes instead of a string, but they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - else: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component, and you " - "are OK with users of your component changing the fingerprinting " - "algorithm through settings, use " - "crawler.request_fingerprinter.fingerprint() instead in your " - "Scrapy component (you can get the crawler object from the " - "'from_crawler' class method).\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "Either way, the resulting fingerprints will be returned as " - "bytes, not as a string, and they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - processed_include_headers: Optional[Tuple[bytes, ...]] = None - if include_headers: - processed_include_headers = tuple( - to_bytes(h.lower()) for h in sorted(include_headers) - ) - cache = _deprecated_fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) - if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if processed_include_headers: - for part in _serialize_headers(processed_include_headers, request): - fp.update(part) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - -def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return bytes.fromhex(request_fingerprint(*args, **kwargs)) - - -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache: ( + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +) _fingerprint_cache = WeakKeyDictionary() @@ -227,13 +116,12 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec return cache[cache_key] class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: @@ -250,42 +138,24 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional["Crawler"] = None): + def __init__(self, crawler: Optional[Crawler] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.6" - if implementation == "2.6": + implementation = "SENTINEL" + + if implementation != "SENTINEL": message = ( - "'2.6' is a deprecated value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n" - "\n" - "It is also the default value. In other words, it is normal " - "to get this warning if you have not defined a value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so " - "for backward compatibility reasons, but it will change in a " - "future version of Scrapy.\n" - "\n" - "See the documentation of the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for " - "information on how to handle this deprecation." + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" + "And it will be removed in future version of Scrapy." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = _request_fingerprint_as_bytes - elif implementation == "2.7": - self._fingerprint = fingerprint - else: - raise ValueError( - f"Got an invalid value on setting " - f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.6' (deprecated) " - f"and '2.7'." - ) + self._fingerprint = fingerprint def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 77d54aff9..a0b06f75c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,24 +2,28 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + +from __future__ import annotations + import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html -import scrapy -from scrapy.http.response import Response from scrapy.utils.python import to_bytes, to_unicode -_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() +if TYPE_CHECKING: + from scrapy.http import Response, TextResponse + +_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() -def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: +def get_base_url(response: TextResponse) -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -29,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) +_metaref_cache: WeakKeyDictionary[ + Response, Union[Tuple[None, None], Tuple[float, str]] +] = WeakKeyDictionary() def get_meta_refresh( - response: "scrapy.http.response.text.TextResponse", + response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" @@ -54,15 +58,37 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" +def _remove_html_comments(body): + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + else: + body = body[:start] + body[end + 3 :] + start = body.find(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) + _remove_html_comments(body) + repl = rf'\0' + body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..89cfbd2ec 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,5 @@ """Helper functions for working with signals""" + import collections.abc import logging from typing import Any as TypingAny @@ -97,7 +98,10 @@ def send_catch_log_deferred( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d.addBoth( + lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + ) dfds.append(d) d = DeferredList(dfds) d.addCallback(lambda out: [x[1] for x in out]) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..cf429043d 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,21 +4,22 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional + +from typing import Any, Dict, Generator, Iterator, Optional, Union from urllib.parse import urljoin -import lxml.etree +import lxml.etree # nosec class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: str): + def __init__(self, xmltext: Union[str, bytes]): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..cbbb01d85 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,18 +34,15 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] - ... +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType) -> Deferred: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable: ... def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: @@ -83,8 +80,7 @@ def spidercls_for_request( default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> Type[Spider]: ... @overload @@ -94,8 +90,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... @overload @@ -105,8 +100,7 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... def spidercls_for_request( diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index e74769c65..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] +import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 709e0b00d..7a8c5c859 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -86,8 +86,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..9d97cb12f 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,6 +5,7 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ + import re from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 9b637bdb0..42e5e9be4 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -4,7 +4,7 @@ from typing import List, Tuple import cryptography import cssselect -import lxml.etree +import lxml.etree # nosec import parsel import twisted import w3lib diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 9ac62c090..13ab501ed 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -140,7 +140,7 @@ Example: The data flow with Spider Middleware v2 is as follows: -1. When a response arrives from the engine, it it passed through all the spider +1. When a response arrives from the engine, it is passed through all the spider middlewares (in descending order). The result of each middleware ``process_response`` is kept and then returned along with the spider callback result diff --git a/setup.py b/setup.py index d6ba4765e..96eacef23 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.4.1", + "defusedxml>=0.7.1", ] extras_require = { ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 23260ab0d..5bf7512bc 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -9,6 +9,7 @@ from twisted.python.runtime import platform from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import configure_logging from tests.mockserver import MockDNSServer, MockServer @@ -30,7 +31,7 @@ class LocalhostSpider(Spider): yield Request(self.url) def parse(self, response): - netloc = urlparse(response.url).netloc + netloc = urlparse_cached(response).netloc host = netloc.split(":")[0] self.logger.info(f"Host: {host}") self.logger.info(f"Type: {type(response.ip_address)}") diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index cdef75f99..61a690cc8 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,50 +1,47 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB -/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT -NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ -u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH -JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB -eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls -90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ -epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 -VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn -v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT -I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU -oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au -/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG -gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn -YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf -dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 -9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC -QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu -bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW -CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR -OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu -PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK -eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx -BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ -B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy -HzPh2taMnCA0h1fNLFFBkw88 ------END PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF +AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9 +rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix +n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ +YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B +qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA +xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3 +H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI +tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG +BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL +QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt +0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW +9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG +UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4 +FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX +G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE +9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc +4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL +tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z +XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB +aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq +VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w +qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR +Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah +zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux +-----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM -CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX -DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt -aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW -ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 -ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg -z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI -ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE -6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ -0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC -AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD -BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG -CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA -A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd -kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe -gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY -1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj -envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj -wiKjdLjCmJgjGAQjRw5VYJ8JI0XO +MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL +BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN +MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv +eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb +Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L +s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja +CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO +wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49 ++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB +/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4 +GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13 +x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o +/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts +L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q +8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d +sTjU4zDdBcBw -----END CERTIFICATE----- diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..bde3de228 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,4 +1,5 @@ """DBM-like dummy module""" + import collections from typing import Any, DefaultDict diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 5b75674f5..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Tests requirements -attrs -pexpect >= 4.8.0 -pyftpdlib >= 1.5.8 -pytest -pytest-cov==4.0.0 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -bpython # optional for shell wrapper tests -brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests -zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.bin new file mode 100644 index 000000000..50059866f --- /dev/null +++ b/tests/sample_data/compressed/bomb-br.bin @@ -0,0 +1,2 @@ +;nުVp SmoY2 +()-д=_o \ No newline at end of file diff --git a/tests/sample_data/compressed/bomb-deflate.bin b/tests/sample_data/compressed/bomb-deflate.bin new file mode 100644 index 000000000..3598aca07 Binary files /dev/null and b/tests/sample_data/compressed/bomb-deflate.bin differ diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin new file mode 100644 index 000000000..64aa0c369 Binary files /dev/null and b/tests/sample_data/compressed/bomb-gzip.bin differ diff --git a/tests/sample_data/compressed/bomb-zstd.bin b/tests/sample_data/compressed/bomb-zstd.bin new file mode 100644 index 000000000..4b0efa8a4 Binary files /dev/null and b/tests/sample_data/compressed/bomb-zstd.bin differ diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 000000000..d66f4c5a0 Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin differ diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 000000000..5066842ef Binary files /dev/null and b/tests/sample_data/compressed/html-gzip-deflate.bin differ diff --git a/tests/spiders.py b/tests/spiders.py index f29dea2a1..ea419afbd 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,8 +1,10 @@ """ Some spiders used for testing and benchmarking """ + import asyncio import time +from typing import Optional from urllib.parse import urlencode from twisted.internet import defer @@ -77,6 +79,28 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class LogSpider(MetaSpider): + name = "log_spider" + + def log_debug(self, message: str, extra: Optional[dict] = None): + self.logger.debug(message, extra=extra) + + def log_info(self, message: str, extra: Optional[dict] = None): + self.logger.info(message, extra=extra) + + def log_warning(self, message: str, extra: Optional[dict] = None): + self.logger.warning(message, extra=extra) + + def log_error(self, message: str, extra: Optional[dict] = None): + self.logger.error(message, extra=extra) + + def log_critical(self, message: str, extra: Optional[dict] = None): + self.logger.critical(message, extra=extra) + + def parse(self, response): + pass + + class SlowSpider(DelaySpider): name = "slow" @@ -301,8 +325,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get("seed")) - for req in super().parse(response): - yield req + yield from super().parse(response) class SingleRequestSpider(MetaSpider): diff --git a/tests/test_addons.py b/tests/test_addons.py index 0f4f2e5b8..f1b01bc5c 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -89,7 +89,7 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) - def test_create_instance(self): + def test_build_from_crawler(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, @@ -167,12 +167,12 @@ class AddonManagerTest(unittest.TestCase): pass with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.create_instance") as create_instance_mock: + with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: settings_dict = { "ADDONS": {LoggedAddon: 1}, } addon = LoggedAddon() - create_instance_mock.return_value = addon + build_from_crawler_mock.return_value = addon crawler = get_crawler(settings_dict=settings_dict) logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 129ef0121..b0f1cd38a 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,8 @@ +import sys +from io import StringIO +from unittest.mock import Mock, PropertyMock, call, patch + +from scrapy.commands.check import Command, TextTestResult from tests.test_commands import CommandTest @@ -16,11 +21,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://toscrape.com'] + start_urls = ['data:,'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://toscrape.com + @url data:, {contracts} \"\"\" {parse_def} @@ -94,3 +99,102 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [] + result.unexpectedSuccesses = ["a", "b"] + with patch.object(result.stream, "write") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_has_calls([call("FAILED"), call("\n")]) + + def test_printSummary_with_unsuccessful_test_result_with_only_failures(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1)") + + def test_printSummary_with_unsuccessful_test_result_with_only_errors(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (errors=1)") + + def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1, errors=1)") + + @patch("scrapy.commands.check.ContractsManager") + def test_run_with_opts_list_prints_spider(self, cm_cls_mock): + output = StringIO() + sys.stdout = output + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + type(spider_cls_mock).name = PropertyMock(return_value=spider_name) + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = ["fakeMethod1", "fakeMethod2"] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=True)) + + self.assertEqual( + "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() + ) + sys.stdout = sys.__stdout__ + + @patch("scrapy.commands.check.ContractsManager") + def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods( + self, cm_cls_mock + ): + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = [] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=False)) + + cmd.crawler_process.crawl.assert_not_called() diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 037333c03..9356d6b79 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider): if i > 5: raise ValueError("Stopping the processing") +class CallbackSignatureDownloaderMiddleware: + def process_request(self, request, spider): + from inspect import signature + spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}") + + class MySpider(scrapy.Spider): name = '{self.spider_name}' + custom_settings = {{ + "DOWNLOADER_MIDDLEWARES": {{ + CallbackSignatureDownloaderMiddleware: 0, + }} + }} + def parse(self, response): if getattr(self, 'test_arg', None): self.logger.debug('It Works!') @@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + log = _textmode(stderr) + self.assertIn("DEBUG: It Works!", log) + self.assertIn( + "DEBUG: request.callback signature: (response, foo=None, key=None)", log + ) @defer.inlineCallbacks def test_request_without_meta(self): diff --git a/tests/test_commands.py b/tests/test_commands.py index 36f800850..b9d468c66 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,13 +6,12 @@ import platform import re import subprocess import sys -import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import mkdtemp +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import Dict, Generator, Optional, Union from unittest import skipIf @@ -82,7 +81,7 @@ class ProjectTest(unittest.TestCase): rmtree(self.temp_path) def call(self, *new_args, **kwargs): - with tempfile.TemporaryFile() as out: + with TemporaryFile() as out: args = (sys.executable, "-m", "scrapy.cmdline") + new_args return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs @@ -988,42 +987,19 @@ class MySpider(scrapy.Spider): self.assertIn("The value of FOO is 42", log) -@skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" def setUp(self): - super().setUp() + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..81cff4947 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", ) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 96d43b2b9..6cde4ed8c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -76,11 +76,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, total, delay, randomize=False): - crawl_kwargs = dict( - maxlatency=delay * 2, - mockserver=self.mockserver, - total=total, - ) + crawl_kwargs = { + "maxlatency": delay * 2, + "mockserver": self.mockserver, + "total": total, + } tolerance = 1 - (0.6 if randomize else 0.2) settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 92bd5f38f..989208694 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -12,12 +12,13 @@ import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy +from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env +# To prevent warnings. +BASE_SETTINGS = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", +} + + +def get_raw_crawler(spidercls=None, settings_dict=None): + """get_crawler alternative that only calls the __init__ method of the + crawler.""" + settings = Settings() + settings.setdict(settings_dict or {}) + return Crawler(spidercls or DefaultSpider, settings) + class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_crawl_twice_deprecated(self): - crawler = Crawler(NoRequestsSpider) + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.warns( ScrapyDeprecationWarning, @@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest): ): yield crawler.crawl() + def test_get_addon(self): + class ParentAddon: + pass + + class TrackingAddon(ParentAddon): + instances = [] + + def __init__(self): + TrackingAddon.instances.append(self) + + def update_settings(self, settings): + pass + + settings = { + **BASE_SETTINGS, + "ADDONS": { + TrackingAddon: 0, + }, + } + crawler = get_crawler(settings_dict=settings) + self.assertEqual(len(TrackingAddon.instances), 1) + expected = TrackingAddon.instances[-1] + + addon = crawler.get_addon(TrackingAddon) + self.assertEqual(addon, expected) + + addon = crawler.get_addon(DefaultSpider) + self.assertIsNone(addon) + + addon = crawler.get_addon(ParentAddon) + self.assertEqual(addon, expected) + + class ChildAddon(TrackingAddon): + pass + + addon = crawler.get_addon(ChildAddon) + self.assertIsNone(addon) + + @inlineCallbacks + def test_get_downloader_middleware(self): + class ParentDownloaderMiddleware: + pass + + class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): + instances = [] + + def __init__(self): + TrackingDownloaderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_downloader_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "DOWNLOADER_MIDDLEWARES": { + TrackingDownloaderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildDownloaderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_downloader_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises( + RuntimeError, crawler.get_downloader_middleware, DefaultSpider + ) + + @inlineCallbacks + def test_get_downloader_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_downloader_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_extension(self): + class ParentExtension: + pass + + class TrackingExtension(ParentExtension): + instances = [] + + def __init__(self): + TrackingExtension.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_extension(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "EXTENSIONS": { + TrackingExtension: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingExtension + yield crawler.crawl() + self.assertEqual(len(TrackingExtension.instances), 1) + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentExtension + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + class ChildExtension(TrackingExtension): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildExtension + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_extension_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + + @inlineCallbacks + def test_get_extension_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_extension(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_item_pipeline(self): + class ParentItemPipeline: + pass + + class TrackingItemPipeline(ParentItemPipeline): + instances = [] + + def __init__(self): + TrackingItemPipeline.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_item_pipeline(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "ITEM_PIPELINES": { + TrackingItemPipeline: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingItemPipeline + yield crawler.crawl() + self.assertEqual(len(TrackingItemPipeline.instances), 1) + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentItemPipeline + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + class ChildItemPipeline(TrackingItemPipeline): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildItemPipeline + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_item_pipeline_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + + @inlineCallbacks + def test_get_item_pipeline_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_item_pipeline(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_spider_middleware(self): + class ParentSpiderMiddleware: + pass + + class TrackingSpiderMiddleware(ParentSpiderMiddleware): + instances = [] + + def __init__(self): + TrackingSpiderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_spider_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "SPIDER_MIDDLEWARES": { + TrackingSpiderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingSpiderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentSpiderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + class ChildSpiderMiddleware(TrackingSpiderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildSpiderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_spider_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + + @inlineCallbacks + def test_get_spider_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_spider_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed(self): runner = self._runner() @@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): runner = self._runner() @@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == "asyncio": CrawlerRunner( @@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() - @defer.inlineCallbacks + @inlineCallbacks def test_shutdown_forced(self): from twisted.internet import reactor @@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, None) yield d p.kill(sig) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 924ece6f9..d3fd63847 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,8 +2,8 @@ import contextlib import os import shutil import sys -import tempfile from pathlib import Path +from tempfile import mkdtemp, mkstemp from typing import Optional, Type from unittest import SkipTest, mock @@ -29,7 +29,7 @@ from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto from tests import NON_EXISTING_RESOLVABLE @@ -107,13 +107,14 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + "^") + self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = create_instance(FileDownloadHandler, None, get_crawler()) + handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): - self.tmpname.unlink() + os.close(self.fd) + os.remove(self.tmpname) def test_download(self): def _test(response): @@ -122,12 +123,12 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b"0123456789") self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(str(self.tmpname))) + request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(path_to_file_uri(self.mktemp())) + request = Request(path_to_file_uri(mkdtemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) @@ -224,8 +225,7 @@ class HttpTestCase(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -257,8 +257,8 @@ class HttpTestCase(unittest.TestCase): else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -557,7 +557,7 @@ class Http11TestCase(HttpTestCase): def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) d = download_handler.download_request(request, Spider("foo")) d.addCallback(lambda r: r.flags) @@ -590,7 +590,7 @@ class Https11TestCase(Http11TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) try: with LogCapture() as log_capture: request = Request(self.getURL("file")) @@ -651,8 +651,7 @@ class Https11CustomCiphers(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) @@ -669,9 +668,7 @@ class Https11CustomCiphers(unittest.TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} ) - self.download_handler = create_instance( - self.download_handler_cls, None, crawler - ) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @defer.inlineCallbacks @@ -751,8 +748,8 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -830,10 +827,9 @@ class S3AnonTestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - self.s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + self.s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, httpdownloadhandler=HttpDownloadHandlerMock, # anon=True, # implicit ) @@ -861,10 +857,9 @@ class S3TestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock, @@ -889,10 +884,9 @@ class S3TestCase(unittest.TestCase): def test_extra_kw(self): try: crawler = get_crawler() - create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + build_from_crawler( + S3DownloadHandler, + crawler, extra_kw=True, ) except Exception as e: @@ -1020,8 +1014,7 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() + self.directory = Path(mkdtemp()) userdir = self.directory / self.username userdir.mkdir() for filename, content in self.test_files: @@ -1039,9 +1032,7 @@ class BaseFTPTestCase(unittest.TestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1099,7 +1090,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() fname_bytes = to_bytes(local_fname) local_fname = Path(local_fname) os.close(f) @@ -1120,7 +1111,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() local_fname = Path(local_fname) os.close(f) meta = {} @@ -1170,9 +1161,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - + self.directory = Path(mkdtemp()) for filename, content in self.test_files: (self.directory / filename).write_bytes(content) @@ -1185,9 +1174,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1197,9 +1184,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): class DataURITestCase(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.download_handler = create_instance( - DataURIDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) self.download_request = self.download_handler.download_request self.spider = Spider("foo") diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 31fa1430d..322075043 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -11,7 +11,7 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( @@ -240,8 +240,8 @@ class Https2ProxyTestCase(Http11ProxyTestCase): interface=self.host, ) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 062e8a8b4..0155c62eb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -22,13 +22,11 @@ class ManagerTestCase(TestCase): self.crawler = get_crawler(Spider, self.settings_dict) self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - # some mw depends on stats collector - self.crawler.stats.open_spider(self.spider) - return self.mwman.open_spider(self.spider) + self.crawler.engine = self.crawler._create_engine() + return self.crawler.engine.open_spider(self.spider, start_requests=()) def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") - return self.mwman.close_spider(self.spider) + return self.crawler.engine.close_spider(self.spider) def _download(self, request, response=None): """Executes downloader mw manager's download method and returns diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 4a81a638e..5eccd396a 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -14,6 +14,8 @@ from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +UNSET = object() + def _cookie_to_set_cookie_value(cookie): """Given a cookie defined as a dictionary with name and value keys, and @@ -320,7 +322,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") + DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) @@ -414,19 +416,19 @@ class CookiesMiddlewareTest(TestCase): "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'value': 'bar'} ('name' is missing)", + " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo'} ('value' is missing)", + " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo', 'value': None} ('value' is missing)", + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", ), ) self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") @@ -732,3 +734,106 @@ class CookiesMiddlewareTest(TestCase): "co.uk", cookies=True, ) + + def _test_cookie_redirect_scheme_change( + self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 + ): + """When a redirect causes the URL scheme to change from *from_scheme* + to *to_scheme*, while domain and port remain the same, and given a + cookie on the initial request with its secure attribute set to + *secure*, check if the cookie should be set on the Cookie header of the + initial request (*cookies1*), if it should be kept by the redirect + middleware (*cookies2*), and if it should be present on the Cookie + header in the redirected request (*cookie3*).""" + cookie_kwargs = {} + if secure is not UNSET: + cookie_kwargs["secure"] = secure + input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + + request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + f"{from_scheme}://a.example", + headers={"Location": f"{to_scheme}://a.example"}, + status=301, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies3 else None) + + def test_cookie_redirect_secure_undefined_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=False, + ) + + def test_cookie_redirect_secure_undefined_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_true_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=False, + cookies3=False, + ) + + def test_cookie_redirect_secure_true_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="http", + to_scheme="https", + cookies1=False, + cookies2=False, + cookies3=True, + ) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index fc110e6cc..500af6536 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -59,7 +59,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") @@ -79,6 +79,6 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a96b710f3..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,15 +1,18 @@ from gzip import GzipFile from io import BytesIO +from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase +from warnings import catch_warnings +from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -24,6 +27,8 @@ FORMAT = { "x-gzip": ("html-gzip.bin", "gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), @@ -34,6 +39,15 @@ FORMAT = { "html-zstd-streaming-no-content-size.bin", "zstd", ), + **{ + f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) + for format_id in ( + "br", # 34 → 11 511 612 + "deflate", # 27 968 → 11 511 612 + "gzip", # 27 988 → 11 511 612 + "zstd", # 1 096 → 11 511 612 + ) + }, } @@ -114,21 +128,12 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_gzip_no_stats(self): - mw = HttpCompressionMiddleware() - response = self._getresponse("gzip") - request = response.request - - self.assertEqual(response.headers["Content-Encoding"], b"gzip") - newresponse = mw.process_response(request, response, self.spider) - self.assertEqual(mw.stats, None) - assert newresponse is not response - assert newresponse.body.startswith(b" body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_setting_br(self): + try: + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_setting("br") + + def test_download_warnsize_setting_deflate(self): + self._test_download_warnsize_setting("deflate") + + def test_download_warnsize_setting_gzip(self): + self._test_download_warnsize_setting("gzip") + + def test_download_warnsize_setting_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") + self._test_download_warnsize_setting("zstd") + + def _test_download_warnsize_spider_attr(self, compression_id): + class DownloadWarnSizeSpider(Spider): + download_warnsize = 10_000_000 + + crawler = get_crawler(DownloadWarnSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr_br(self): + try: + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_spider_attr("br") + + def test_download_warnsize_spider_attr_deflate(self): + self._test_download_warnsize_spider_attr("deflate") + + def test_download_warnsize_spider_attr_gzip(self): + self._test_download_warnsize_spider_attr("gzip") + + def test_download_warnsize_spider_attr_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") + self._test_download_warnsize_spider_attr("zstd") + + def _test_download_warnsize_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_warnsize"] = 10_000_000 + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta_br(self): + try: + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_request_meta("br") + + def test_download_warnsize_request_meta_deflate(self): + self._test_download_warnsize_request_meta("deflate") + + def test_download_warnsize_request_meta_gzip(self): + self._test_download_warnsize_request_meta("gzip") + + def test_download_warnsize_request_meta_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") + self._test_download_warnsize_request_meta("zstd") + + +class HttpCompressionSubclassTest(TestCase): + def test_init_missing_stats(self): + class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): + def __init__(self): + super().__init__() + + crawler = get_crawler(Spider) + with catch_warnings(record=True) as caught_warnings: + HttpCompressionMiddlewareSubclass.from_crawler(crawler) + messages = tuple( + str(warning.message) + for warning in caught_warnings + if warning.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'crawler' parameter " + "or reimplement their 'from_crawler' method." + ), + ), + ) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py new file mode 100644 index 000000000..d4669f450 --- /dev/null +++ b/tests/test_downloadermiddleware_offsite.py @@ -0,0 +1,184 @@ +import pytest + +from scrapy import Request, Spider +from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.test import get_crawler + +UNSET = object() + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_process_request_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.process_request(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_process_request_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + else: + assert mw.process_request(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_process_request_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.process_request(request, spider) is None + + +def test_process_request_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.process_request(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.request_scheduled(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_request_scheduled_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + else: + assert mw.request_scheduled(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_request_scheduled_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.request_scheduled(request, spider) is None + + +def test_request_scheduled_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.request_scheduled(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index dc15b672c..4bfd34fe2 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,9 @@ import unittest +from itertools import chain, product +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.downloadermiddlewares.redirect import ( MetaRefreshMiddleware, RedirectMiddleware, @@ -7,22 +11,1030 @@ from scrapy.downloadermiddlewares.redirect import ( from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import set_environ from scrapy.utils.test import get_crawler -class RedirectMiddlewareTest(unittest.TestCase): +class Base: + class Test(unittest.TestCase): + def test_priority_adjust(self): + req = Request("http://a.com") + rsp = self.get_response(req, "http://a.com/redirected") + req2 = self.mw.process_response(req, rsp, self.spider) + self.assertGreater(req2.priority, req.priority) + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, "GET") + assert ( + "Content-Type" not in req2.headers + ), "Content-Type header must not be present in redirected request" + assert ( + "Content-Length" not in req2.headers + ), "Content-Length header must not be present in redirected request" + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://scrapytest.org/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + self.assertEqual(req.meta["redirect_times"], 1) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_redirect_urls(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req1, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.url, "http://scrapytest.org/redirected") + self.assertEqual( + req2.meta["redirect_urls"], ["http://scrapytest.org/first"] + ) + self.assertEqual(req3.url, "http://scrapytest.org/redirected2") + self.assertEqual( + req3.meta["redirect_urls"], + ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], + ) + + def test_redirect_reasons(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + self.assertEqual(req2.meta["redirect_reasons"], [self.reason]) + self.assertEqual(req3.meta["redirect_reasons"], [self.reason, self.reason]) + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual( + original_request.headers, internal_redirect_request.headers + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response, self.spider + ) + self.assertIsInstance(http_redirect_request, Request) + self.assertEqual(http_request.headers, http_redirect_request.headers) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response, self.spider + ) + self.assertIsInstance(to_explicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_explicit_port_redirect_request.headers + ) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response, self.spider + ) + self.assertIsInstance(to_implicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_implicit_port_redirect_request.headers + ) + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response, self.spider + ) + self.assertIsInstance(different_port_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + different_port_redirect_request.headers.to_unicode_dict(), + ) + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response, self.spider + ) + self.assertIsInstance(upgrade_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + upgrade_redirect_request.headers.to_unicode_dict(), + ) + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response, self.spider + ) + self.assertIsInstance(downgrade_redirect_request, Request) + self.assertEqual( + safe_headers, + downgrade_redirect_request.headers.to_unicode_dict(), + ) + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + +class RedirectMiddlewareTest(Base.Test): + mwcls = RedirectMiddleware + reason = 302 + def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") - self.mw = RedirectMiddleware.from_crawler(self.crawler) + self.mw = self.mwcls.from_crawler(self.crawler) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = Response( - "http://a.com", headers={"Location": "http://a.com/redirected"}, status=301 - ) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location, status=302): + headers = {"Location": location} + return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): def _test(method, status=301): @@ -52,51 +1064,6 @@ class RedirectMiddlewareTest(unittest.TestCase): _test("POST", status=308) _test("HEAD", status=308) - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = Response(url, headers={"Location": url2}, status=301) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = Response(url2, status=200) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - def test_redirect_302(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = Response(url, headers={"Location": url2}, status=302) - - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -108,10 +1075,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url2) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_relative(self): url = "http://www.example.com/302" url2 = "///i8n.example2.com/302" @@ -124,81 +1087,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url3) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/302") - rsp = Response( - "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302 - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = Response( - "http://www.scrapytest.org/302", - headers={"Location": "/redirected"}, - status=302, - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected"}, - status=302, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected", - headers={"Location": "/redirected2"}, - status=302, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected1"}, - status=301, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected1", - headers={"Location": "/redirected2"}, - status=301, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], [301]) - self.assertEqual(req3.meta["redirect_reasons"], [301, 301]) - def test_spider_handling(self): smartspider = self.crawler._create_spider("smarty") smartspider.handle_httpstatus_list = [404, 301, 302] @@ -247,22 +1135,84 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_no_location(self): + request = Request("https://example.com") + response = Response(request.url, status=302) + assert self.mw.process_response(request, response, self.spider) is response + + +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) +def test_redirect_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = RedirectMiddleware.from_crawler(crawler) + request = Request(url) + response = Response(url, headers={"Location": location}, status=301) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class MetaRefreshMiddlewareTest(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" -class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("foo") - self.mw = MetaRefreshMiddleware.from_crawler(crawler) + self.mw = self.mwcls.from_crawler(crawler) def _body(self, interval=5, url="http://example.org/newpage"): - html = f"""""" - return html.encode("utf-8") + return meta_refresh_body(url, interval) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): req = Request(url="http://example.org") @@ -301,62 +1251,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): ), "Content-Length header must not be present in redirected request" assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/max") - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected")) - req2 = self.mw.process_response(req1, rsp1, self.spider) - assert isinstance(req2, Request), req2 - rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2")) - req3 = self.mw.process_response(req2, rsp2, self.spider) - assert isinstance(req3, Request), req3 - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse( - "http://scrapytest.org/first", body=self._body(url="/redirected") - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = HtmlResponse( - "http://scrapytest.org/redirected", body=self._body(url="/redirected1") - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"]) - self.assertEqual( - req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"] - ) - def test_ignore_tags_default(self): req = Request(url="http://example.org") body = ( @@ -364,9 +1258,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) def test_ignore_tags_1_x_list(self): """Test that Scrapy 1.x behavior remains possible""" @@ -383,5 +1276,45 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): assert isinstance(response, Response) +@pytest.mark.parametrize( + SCHEME_PARAMS, + ( + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ), +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..ea8c5b4f0 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -3,8 +3,10 @@ import time from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MetaSpider @@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, @@ -70,3 +73,27 @@ class CrawlTestCase(TestCase): } self.assertTrue(max(list(error_delta.values())) < tolerance) + + +def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(settings_dict=settings) + downloader = Downloader(crawler) + downloader._slot_gc_loop.stop() # Prevent an unclean reactor. + request = Request("https://example.com") + _, actual = downloader._get_slot(request, spider=None) + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr( + actual, param + ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d7afb6a1..33544e8db 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -15,8 +15,10 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass +from logging import DEBUG from pathlib import Path from threading import Timer +from unittest.mock import Mock from urllib.parse import urlparse import attr @@ -27,11 +29,13 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider +from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.scheduler import BaseScheduler +from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor +from scrapy.signals import request_scheduled from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -467,6 +471,38 @@ class EngineTest(unittest.TestCase): self.assertNotIn(b"Traceback", stderr) +def test_request_scheduled_signal(caplog): + class TestScheduler(BaseScheduler): + def __init__(self): + self.enqueued = [] + + def enqueue_request(self, request: Request) -> bool: + self.enqueued.append(request) + return True + + def signal_handler(request: Request, spider: Spider) -> None: + if "drop" in request.url: + raise IgnoreRequest + + spider = TestSpider() + crawler = get_crawler(spider.__class__) + engine = ExecutionEngine(crawler, lambda _: None) + engine.downloader._slot_gc_loop.stop() + scheduler = TestScheduler() + engine.slot = Slot((), None, Mock(), scheduler) + crawler.signals.connect(signal_handler, request_scheduled) + keep_request = Request("https://keep.example") + engine._schedule_request(keep_request, spider) + drop_request = Request("https://drop.example") + caplog.set_level(DEBUG) + engine._schedule_request(drop_request, spider) + assert scheduler.enqueued == [ + keep_request + ], f"{scheduler.enqueued!r} != [{keep_request!r}]" + assert "dropped request " in caplog.text + crawler.signals.disconnect(signal_handler, request_scheduled) + + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..fa9389044 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, "John\xa3") ie = self._get_exporter(fields_to_export={"name": "名稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")] + ) def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") @@ -150,7 +152,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) @@ -183,7 +185,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=[i1]) + i2 = {"name": "Maria", "age": [i1]} i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) @@ -371,7 +373,7 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name="John", friends=[4, 8]), + item={"name": "John", "friends": [4, 8]}, include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -386,14 +388,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", @@ -453,8 +455,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name="foo\xa3hoo", age="22") - i2 = dict(name="bar", age=i1) + i1 = {"name": "foo\xa3hoo", "age": "22"} + i2 = {"name": "bar", "age": i1} i3 = self.item_class(name="buz", age=i2) self.assertExportResult( @@ -476,8 +478,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name="foo") - i2 = dict(name="bar", v2={"egg": ["spam"]}) + i1 = {"name": "foo"} + i2 = {"name": "bar", "v2": {"egg": ["spam"]}} i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( @@ -532,7 +534,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -620,9 +622,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name="Joseph\xa3", age="22") + i1 = {"name": "Joseph\xa3", "age": "22"} i2 = self.item_class(name="Maria", age=i1) - i3 = dict(name="Jesus", age=i2) + i3 = {"name": "Jesus", "age": i2} self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 502ada6be..b7312bbcd 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -67,7 +67,7 @@ def extension(settings=None): class TestPeriodicLog(unittest.TestCase): def test_extension_enabled(self): - # Expected that settings for this extension loaded succesfully + # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py new file mode 100644 index 000000000..722a05c26 --- /dev/null +++ b/tests/test_extension_throttle.py @@ -0,0 +1,343 @@ +from logging import INFO +from unittest.mock import Mock + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.throttle import AutoThrottle +from scrapy.http.response import Response +from scrapy.settings.default_settings import ( + AUTOTHROTTLE_MAX_DELAY, + AUTOTHROTTLE_START_DELAY, + DOWNLOAD_DELAY, +) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler as _get_crawler + +UNSET = object() + + +class TestSpider(Spider): + name = "test" + + +def get_crawler(settings=None, spidercls=None): + settings = settings or {} + settings["AUTOTHROTTLE_ENABLED"] = True + return _get_crawler(settings_dict=settings, spidercls=spidercls) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, False), + (False, False), + (True, True), + ), +) +def test_enabled(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_ENABLED"] = value + crawler = _get_crawler(settings_dict=settings) + if expected: + build_from_crawler(AutoThrottle, crawler) + else: + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + "value", + ( + 0.0, + -1.0, + ), +) +def test_target_concurrency_invalid(value): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} + crawler = get_crawler(settings) + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + ("spider", "setting", "expected"), + ( + (UNSET, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), + (UNSET, 1.0, 1.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), + ), +) +def test_mindelay_definition(spider, setting, expected): + settings = {} + if setting is not UNSET: + settings["DOWNLOAD_DELAY"] = setting + + class _TestSpider(Spider): + name = "test" + + if spider is not UNSET: + _TestSpider.download_delay = spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(_TestSpider()) + assert at.mindelay == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, AUTOTHROTTLE_MAX_DELAY), + (1.0, 1.0), + ), +) +def test_maxdelay_definition(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_MAX_DELAY"] = value + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(TestSpider()) + assert at.maxdelay == expected + + +@pytest.mark.parametrize( + ("min_spider", "min_setting", "start_setting", "expected"), + ( + (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ), +) +def test_startdelay_definition(min_spider, min_setting, start_setting, expected): + settings = {} + if min_setting is not UNSET: + settings["DOWNLOAD_DELAY"] = min_setting + if start_setting is not UNSET: + settings["AUTOTHROTTLE_START_DELAY"] = start_setting + + class _TestSpider(Spider): + name = "test" + + if min_spider is not UNSET: + _TestSpider.download_delay = min_spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + spider = _TestSpider() + at._spider_opened(spider) + assert spider.download_delay == expected + + +@pytest.mark.parametrize( + ("meta", "slot", "throttle"), + ( + ({}, None, None), + ({"download_latency": 1.0}, None, None), + ({"download_slot": "foo"}, None, None), + ({"download_slot": "foo"}, "foo", None), + ({"download_latency": 1.0, "download_slot": "foo"}, None, None), + ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), + ), +) +def test_skipped(meta, slot, throttle): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + request = Request("https://example.com", meta=meta) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + if slot is not None: + _slot = Mock() + _slot.throttle = throttle + crawler.engine.downloader.slots[slot] = _slot + at._adjust_delay = None # Raise exception if called. + + at._response_downloaded(None, request, spider) + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 0.75), + (4.0, 2.0, 1.0, 2.0), + (2.0, 1.0, 1.0, 2.0), + (2.0, 4.0, 1.0, 0.75), + (2.0, 2.0, 0.5, 1.0), + (2.0, 2.0, 2.0, 1.5), + ), +) +def test_adjustment(download_latency, target_concurrency, slot_delay, expected): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("mindelay", "maxdelay", "expected"), + ( + (0.5, 2.0, 1.0), + (0.25, 0.5, 0.5), + (2.0, 4.0, 2.0), + ), +) +def test_adjustment_limits(mindelay, maxdelay, expected): + download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) + # expected adjustment without limits with these values: 1.0 + settings = { + "AUTOTHROTTLE_MAX_DELAY": maxdelay, + "AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency, + "DOWNLOAD_DELAY": mindelay, + } + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 + (4.0, 2.0, 1.0, 2.0), + ), +) +def test_adjustment_bad_response( + download_latency, target_concurrency, slot_delay, expected +): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, status=400) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +def test_debug(caplog): + settings = {"AUTOTHROTTLE_DEBUG": True} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [ + ( + "scrapy.extensions.throttle", + INFO, + "slot: foo | conc: 2 | delay: 1500 ms (-500) | latency: 1000 ms | size: 3 bytes", + ), + ] + + +def test_debug_disabled(caplog): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 89169fd7c..d7560b5ff 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items data = yield self.run_and_export(TestSpider, settings) return data @@ -1732,6 +1731,7 @@ class FeedExportTest(FeedExportTestBase): def store(self, file): Storage.store_file = file + Storage.file_was_closed = file.closed file.close() settings = { @@ -1747,6 +1747,7 @@ class FeedExportTest(FeedExportTestBase): } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) + self.assertFalse(Storage.file_was_closed) class FeedPostProcessedExportsTest(FeedExportTestBase): @@ -2300,7 +2301,7 @@ class BatchDeliveriesTest(FeedExportTestBase): content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() - defer.returnValue(content) + return content @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -2696,8 +2697,7 @@ class BatchDeliveriesTest(FeedExportTestBase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items with MockServer() as server: TestSpider.start_urls = [server.url("/")] diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8fdf3d56f..995c02a1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from tempfile import mkdtemp from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -185,8 +186,7 @@ class Https2ClientProtocolTestCase(TestCase): certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): - self.temp_directory = self.mktemp() - Path(self.temp_directory).mkdir() + self.temp_directory = mkdtemp() r = File(self.temp_directory) r.putChild(b"get-data-html-small", GetDataHtmlSmall()) r.putChild(b"get-data-html-large", GetDataHtmlLarge()) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 9e43b72b0..932644320 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,8 @@ from unittest import TestCase -from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.utils.httpobj import urlparse_cached class WrappedRequestTest(TestCase): @@ -17,12 +17,12 @@ class WrappedRequestTest(TestCase): self.assertEqual(self.wrapped.full_url, self.request.url) def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc) - self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc) + self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc) + self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc) def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme) - self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme) + self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme) + self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme) def test_is_unverifiable(self): self.assertFalse(self.wrapped.is_unverifiable()) @@ -43,6 +43,13 @@ class WrappedRequestTest(TestCase): def test_get_header(self): self.assertEqual(self.wrapped.get_header("content-type"), "text/html") self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") + self.assertEqual(self.wrapped.get_header("xxxxx"), None) + wrapped = WrappedRequest( + Request( + "http://www.example.com/page.html", headers={"empty-binary-header": b""} + ) + ) + self.assertEqual(wrapped.get_header("empty-binary-header"), "") def test_header_items(self): self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..7ce73e6ff 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -5,7 +5,7 @@ import warnings import xmlrpc.client from typing import Any, Dict, List from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes, urlparse +from urllib.parse import parse_qs, unquote_to_bytes from scrapy.http import ( FormRequest, @@ -16,6 +16,7 @@ from scrapy.http import ( XmlRpcRequest, ) from scrapy.http.request import NO_CALLBACK +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode @@ -617,8 +618,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse(req.url).hostname, "www.example.com") - self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") + self.assertEqual(urlparse_cached(req).hostname, "www.example.com") + self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -666,8 +667,8 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse(r1.url).hostname, "www.example.com") - self.assertEqual(urlparse(r1.url).path, "/this/get.php") + self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") + self.assertEqual(urlparse_cached(r1).path, "/this/get.php") fs = _qs(r1) self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) @@ -1426,6 +1427,58 @@ class FormRequestTest(RequestTest): r = self.request_class.from_response(response) self.assertEqual(r.method, expected) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ + + + + """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=("a",)) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1642,6 +1695,25 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_replacement_both_body_and_data_warns(self): + """Test that we get a warning if both body and data are passed""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with warnings.catch_warnings(record=True) as _warnings: + r1.replace(data=data2, body=body2) + self.assertIn( + "Both body and data passed. data will be ignored", + str(_warnings[0].message), + ) + def tearDown(self): warnings.resetwarnings() super().tearDown() diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..daf5d4f59 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__( + self, *args, **kwargs + ): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..d9c09a16a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -37,7 +37,7 @@ class Base: page4_url = "http://example.com/page%204.html" self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -55,7 +55,7 @@ class Base: def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -70,7 +70,7 @@ class Base: def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -93,7 +93,7 @@ class Base: def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -116,7 +116,7 @@ class Base: def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -127,7 +127,7 @@ class Base: def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -137,7 +137,7 @@ class Base: def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -148,7 +148,7 @@ class Base: lx = self.extractor_cls(allow="sample") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -162,7 +162,7 @@ class Base: lx = self.extractor_cls(allow="sample", deny="3") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -171,7 +171,7 @@ class Base: lx = self.extractor_cls(allow_domains="google.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -179,7 +179,7 @@ class Base: lx = self.extractor_cls(deny_domains="example.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -265,7 +265,7 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -337,7 +337,7 @@ class Base: restrict_css=("#subwrapper + a",), ) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -705,7 +705,7 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -745,6 +745,118 @@ class Base: lx = self.extractor_cls() self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + def test_link_extractor_aggregation(self): + """When a parameter like restrict_css is used, the underlying + implementation calls its internal link extractor once per selector + matching the specified restrictions, and then aggregates the + extracted links. + + Test that aggregation respects the unique and canonicalize + parameters. + """ + # unique=True (default), canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",)) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=True (default), canonicalize=True + lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ], + ) + + # unique=False, canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",), unique=False) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=False, canonicalize=True + lx = self.extractor_cls( + restrict_css=("div",), unique=False, canonicalize=True + ) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ], + ) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor @@ -758,7 +870,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", text="Item 1", nofollow=False @@ -779,7 +891,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -791,7 +903,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -803,7 +915,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -818,9 +930,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - @mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( @@ -837,7 +946,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -851,3 +960,15 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ), ], ) + + def test_link_allowed_is_false_with_empty_url(self): + bad_link = Link("") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + + def test_link_allowed_is_false_with_bad_url_prefix(self): + bad_link = Link("htp://should_be_http.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + + def test_link_allowed_is_false_with_missing_url_prefix(self): + bad_link = Link("should_have_prefix.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d7f773d5c..528efa142 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -565,37 +565,37 @@ class NoInputReprocessingFromDictTest(unittest.TestCase): """ def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title="foo")) + il = NoInputReprocessingDictLoader(item={"title": "foo"}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) + il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingDictLoader() il.add_value("title", "foo") il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingDictLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) @@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] + for k, v in self.test_list_equals.items(): + expr, test_list, expected = v test = SelectJmes(expr)(test_list) self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{k}" got {test} expected {expected}' ) diff --git a/tests/test_logstats.py b/tests/test_logstats.py new file mode 100644 index 000000000..d87285df7 --- /dev/null +++ b/tests/test_logstats.py @@ -0,0 +1,62 @@ +import unittest +from datetime import datetime + +from scrapy.extensions.logstats import LogStats +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider + + +class TestLogStats(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(SimpleSpider) + self.spider = self.crawler._create_spider("spidey") + self.stats = self.crawler.stats + + self.stats.set_value("response_received_count", 4802) + self.stats.set_value("item_scraped_count", 3201) + + def test_stats_calculations(self): + logstats = LogStats.from_crawler(self.crawler) + + with self.assertRaises(AttributeError): + logstats.pagesprev + logstats.itemsprev + + logstats.spider_opened(self.spider) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + logstats.calculate_stats() + self.assertEqual(logstats.items, 3201) + self.assertEqual(logstats.pages, 4802) + self.assertEqual(logstats.irate, 0.0) + self.assertEqual(logstats.prate, 0.0) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + # Simulate what happens after a minute + self.stats.set_value("response_received_count", 5187) + self.stats.set_value("item_scraped_count", 3492) + logstats.calculate_stats() + self.assertEqual(logstats.items, 3492) + self.assertEqual(logstats.pages, 5187) + self.assertEqual(logstats.irate, 291.0) + self.assertEqual(logstats.prate, 385.0) + self.assertEqual(logstats.pagesprev, 5187) + self.assertEqual(logstats.itemsprev, 3492) + + # Simulate when spider closes after running for 30 mins + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + logstats.spider_closed(self.spider, "test reason") + self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) + self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) + + def test_stats_calculations_no_time(self): + """The stat values should be None since the start and finish time are + not available. + """ + logstats = LogStats.from_crawler(self.crawler) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute")) diff --git a/tests/test_mail.py b/tests/test_mail.py index 2535e58db..ff1505397 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -91,7 +91,7 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(attach.get_payload(decode=True), b"content") def _catch_mail_sent(self, **kwargs): - self.catched_msg = dict(**kwargs) + self.catched_msg = {**kwargs} def test_send_utf8(self): subject = "sübjèçt" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index ed8483483..5a9a217ce 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from tempfile import mkdtemp from typing import Optional, Set from testfixtures import LogCapture @@ -9,6 +10,7 @@ from w3lib.url import add_or_replace_parameter from scrapy import signals from scrapy.crawler import CrawlerRunner +from scrapy.utils.misc import load_object from tests.mockserver import MockServer from tests.spiders import SimpleSpider @@ -66,8 +68,7 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = Path(self.mktemp()) - self.tmpmediastore.mkdir() + self.tmpmediastore = Path(mkdtemp()) self.settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, @@ -139,7 +140,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) + self.assertEqual(list(self.tmpmediastore.iterdir()), []) @defer.inlineCallbacks def test_download_media(self): @@ -193,6 +194,29 @@ class FileDownloadCrawlTestCase(TestCase): crawler.stats.get_value("downloader/response_status_count/302"), 3 ) + @defer.inlineCallbacks + def test_download_media_file_path_error(self): + cls = load_object(self.pipeline_class) + + class ExceptionRaisingMediaPipeline(cls): + def file_path(self, request, response=None, info=None, *, item=None): + return 1 / 0 + + settings = { + **self.settings, + "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, + } + runner = CrawlerRunner(settings) + crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + with LogCapture() as log: + yield crawler.crawl( + self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key, + mockserver=self.mockserver, + ) + self.assertIn("ZeroDivisionError", str(log)) + skip_pillow: Optional[str] try: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e7000e314..0babde4d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -221,7 +221,7 @@ class FilesPipelineTestCase(unittest.TestCase): file_path = CustomFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ).file_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual(file_path(request, item=item), "full/path-to-store-file") diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2e2e06b89..18a2454b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -132,7 +132,7 @@ class ImagesPipelineTestCase(unittest.TestCase): thumb_path = CustomImagesPipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ).thumb_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual( thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" @@ -433,14 +433,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ] # This should match what is defined in ImagesPipeline. - default_pipeline_settings = dict( - MIN_WIDTH=0, - MIN_HEIGHT=0, - EXPIRES=90, - THUMBS={}, - IMAGES_URLS_FIELD="image_urls", - IMAGES_RESULT_FIELD="images", - ) + default_pipeline_settings = { + "MIN_WIDTH": 0, + "MIN_HEIGHT": 0, + "EXPIRES": 90, + "THUMBS": {}, + "IMAGES_URLS_FIELD": "image_urls", + "IMAGES_RESULT_FIELD": "images", + } def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..d4dde4a40 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + skip_pillow: Optional[str] = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None @@ -59,7 +59,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_to_download(request, self.info) is None def test_default_get_media_requests(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.get_media_requests(item, self.info) is None def test_default_media_downloaded(self): @@ -73,7 +73,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_failed(fail, request, self.info) is fail def test_default_item_completed(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default @@ -98,7 +98,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): @inlineCallbacks def test_default_process_item(self): - item = dict(name="name") + item = {"name": "name"} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item @@ -226,11 +226,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): rsp = Response("http://url1") req = Request( "http://url1", - meta=dict(response=rsp), + meta={"response": rsp}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp)]) self.assertEqual( @@ -250,11 +250,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): fail = Failure(Exception()) req = Request( "http://url1", - meta=dict(response=fail), + meta={"response": fail}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(False, fail)]) self.assertEqual( @@ -272,10 +272,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) fail = Failure(Exception()) - req2 = Request("http://url2", meta=dict(response=fail)) - item = dict(requests=[req1, req2]) + req2 = Request("http://url2", meta={"response": fail}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled @@ -294,7 +294,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") - item = dict(requests=req) # pass a single item + item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item self.assertIn(self.fingerprint(req), self.info.downloaded) @@ -302,7 +302,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = dict(requests=iter([req1, req2])) + item = {"requests": iter([req1, req2])} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded @@ -311,17 +311,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - item = dict(requests=req1) + req1 = Request("http://url1", meta={"response": rsp1}) + item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1)]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=req2) + item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) @@ -330,11 +330,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=[req1, req2]) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @@ -359,16 +359,16 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def rsp2_func(): self.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request("http://url", meta=dict(response=rsp1_func)) - req2 = Request(req1.url, meta=dict(response=rsp2_func)) - item = dict(requests=[req1, req2]) + req1 = Request("http://url", meta={"response": rsp1_func}) + req2 = Request(req1.url, meta={"response": rsp2_func}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) - item = dict(requests=req) + req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, "ITSME")]) self.assertEqual( diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index dc0a82086..93f006c76 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -27,10 +27,11 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" + cert_path = Path(__file__).parent.resolve() / "keys" self.proc = Popen( [ sys.executable, + "-u", "-c", script, "--listen-host", @@ -39,14 +40,14 @@ sys.exit(mitmdump()) "0", "--proxyauth", f"{self.auth_user}:{self.auth_pass}", - "--certs", - str(cert_path), + "--set", + f"confdir={cert_path}", "--ssl-insecure", ], stdout=PIPE, ) line = self.proc.stdout.readline().decode("utf-8") - host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) + host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" return address diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 6e1ed82f0..2633cca5b 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,6 +1,13 @@ import unittest -from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse +from scrapy.http import ( + Headers, + HtmlResponse, + JsonResponse, + Response, + TextResponse, + XmlResponse, +) from scrapy.responsetypes import responsetypes @@ -26,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase): ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), - ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), + ( + "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), + XmlResponse, + ), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) @@ -40,8 +50,9 @@ class ResponseTypesTest(unittest.TestCase): ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse), ("application/xml; charset=UTF-8", XmlResponse), ("application/octet-stream", Response), - ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse), - ("application/json-amazonui-streaming;charset=UTF-8", TextResponse), + ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse), + ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse), + ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse), (b"application/x-download; filename=\x80dummy.txt", Response), ] for source, cls in mappings: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index d7a923085..6ad30deed 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,5 +1,7 @@ from twisted.trial import unittest +from scrapy.robotstxt import decode_robotstxt + def reppy_available(): # check if reppy parser is installed @@ -141,6 +143,25 @@ class BaseRobotParserTest: ) +class DecodeRobotsTxtTest(unittest.TestCase): + def test_native_string_conversion(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt( + robotstxt_body, spider=None, to_native_str_type=True + ) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_utf8(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_non_utf8(self): + robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f8465a5ff..02b50baa3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -25,7 +25,7 @@ class MockDownloader: def __init__(self): self.slots = {} - def _get_slot_key(self, request, spider): + def get_slot_key(self, request): if Downloader.DOWNLOAD_SLOT in request.meta: return request.meta[Downloader.DOWNLOAD_SLOT] @@ -45,15 +45,15 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", - SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue", - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter", - REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7", - ) + settings = { + "SCHEDULER_DEBUG": False, + "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, + "JOBDIR": jobdir, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -273,14 +273,14 @@ class DownloaderAwareSchedulerTestMixin: while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) downloader.decrement(slot) self.assertTrue( @@ -338,10 +338,10 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): class TestIncompatibility(unittest.TestCase): def _incompatible(self): - settings = dict( - SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", - CONCURRENT_REQUESTS_PER_IP=1, - ) + settings = { + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", + "CONCURRENT_REQUESTS_PER_IP": 1, + } crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name="spider") diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 76ca777a8..5db2e4e50 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,6 +1,6 @@ from typing import Dict, Optional from unittest import TestCase -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from testfixtures import LogCapture from twisted.internet import defer @@ -9,6 +9,7 @@ from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -57,7 +58,7 @@ class TestSpider(Spider): self.start_urls = map(mockserver.url, PATHS) def parse(self, response): - return {"path": urlparse(response.url).path} + return {"path": urlparse_cached(response).path} class InterfaceCheckMixin: diff --git a/tests/test_selector.py b/tests/test_selector.py index 85527bba9..1b5f3f018 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -80,10 +80,10 @@ class SelectorTestCase(unittest.TestCase): meta = ( '' ) - head = "" + meta + "" + head = f"{meta}" body_content = '\xa3' - body = "" + body_content + "" - html = "" + head + body + "" + body = f"{body_content}" + html = f"{head}{body}" encoding = "utf-8" html_utf8 = html.encode(encoding) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index e7799737f..9ee248538 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase): mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object self.assertEqual(mydict.getpriority("key"), 0) @mock.patch("scrapy.settings.default_settings", default_settings) @@ -440,7 +440,7 @@ class SettingsTest(unittest.TestCase): def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler - from scrapy.utils.misc import create_instance + from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler class TestPipeline: @@ -468,7 +468,7 @@ class SettingsTest(unittest.TestCase): myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") self.assertEqual(myhandler, FileDownloadHandler) - myhandler_instance = create_instance(myhandler, None, get_crawler()) + myhandler_instance = build_from_crawler(myhandler, get_crawler()) self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..d629d33af 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,6 +2,8 @@ import gzip import inspect import warnings from io import BytesIO +from logging import WARNING +from pathlib import Path from typing import Any from unittest import mock @@ -25,7 +27,7 @@ from scrapy.spiders import ( ) from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests import get_testdata, tests_datadir class SpiderTest(unittest.TestCase): @@ -149,10 +151,10 @@ class XMLFeedSpiderTest(SpiderTest): body = b""" - http://www.example.com/Special-Offers.html2009-08-16 + http://www.example.com/Special-Offers.html2009-08-16 - http://www.example.com/2009-08-16 + http://www.example.com/2009-08-16 """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) @@ -317,8 +319,7 @@ class CrawlSpiderTest(SpiderTest): rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) def dummy_process_links(self, links): - for link in links: - yield link + yield from links spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) @@ -489,7 +490,8 @@ class SitemapSpiderTest(SpiderTest): GZBODY = f.getvalue() def assertSitemapBody(self, response, body): - spider = self.spider_class("example.com") + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") self.assertEqual(spider._get_sitemap_body(response), body) def test_get_sitemap_body(self): @@ -507,6 +509,7 @@ class SitemapSpiderTest(SpiderTest): url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), ) self.assertSitemapBody(r, self.BODY) @@ -515,7 +518,11 @@ class SitemapSpiderTest(SpiderTest): self.assertSitemapBody(r, self.BODY) def test_get_sitemap_body_xml_url_compressed(self): - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) self.assertSitemapBody(r, self.BODY) # .xml.gz but body decoded by HttpCompression middleware already @@ -692,6 +699,116 @@ Sitemap: /sitemap-relative-url.xml ["http://www.example.com/sitemap2.xml"], ) + def test_compression_bomb_setting(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + class DeprecationTest(unittest.TestCase): def test_crawl_spider(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 04025d30d..f950739f2 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import sys import tempfile import warnings from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -139,8 +140,7 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(self.mktemp()) - self.tmpdir.mkdir() + self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index d167adbb7..38ca8d950 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): class ProcessSpiderOutputSimpleMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result class ProcessSpiderOutputAsyncGenMiddleware: @@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware: class ProcessSpiderOutputUniversalMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result async def process_spider_output_async(self, response, result, spider): async for r in result: @@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): class ProcessStartRequestsSimpleMiddleware: def process_start_requests(self, start_requests, spider): - for r in start_requests: - yield r + yield from start_requests class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ea45b7698..837f1c2c8 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,10 +16,10 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict( - name="foo", - allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], - ) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -50,7 +50,7 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): def _get_spiderargs(self): - return dict(name="foo", allowed_domains=None) + return {"name": "foo", "allowed_domains": None} def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -61,13 +61,16 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spiderargs(self): - return dict(name="foo") + return {"name": "foo"} class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spiderargs(self): bad_hostname = urlparse("http:////scrapytest.org").hostname - return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", None, bad_hostname], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f97125b76..59d18d92e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,6 +1,6 @@ import shutil from datetime import datetime, timezone -from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest @@ -12,8 +12,7 @@ from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): def test_store_load(self): - jobdir = self.mktemp() - Path(jobdir).mkdir() + jobdir = mkdtemp() try: spider = Spider(name="default") dt = datetime.now(tz=timezone.utc) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b444c32b7..499ca46b8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -25,7 +25,7 @@ class BaseQueueTestCase(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() - self.qdir = self.mkdtemp() + self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) def tearDown(self): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..be5c6de81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache for _ in range(max // 2): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index bb0ebc2a4..a7d54b565 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase): class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in range(10): - yield x + yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..ec377bb19 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,14 +1,14 @@ -from pytest import mark +import pytest from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) - +class XmliterBaseTestCase: + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -40,6 +40,7 @@ class XmliterTestCase(unittest.TestCase): attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -53,6 +54,7 @@ class XmliterTestCase(unittest.TestCase): ] self.assertEqual(nodenames, [["matchme..."]]) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -112,6 +114,7 @@ class XmliterTestCase(unittest.TestCase): [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -123,6 +126,7 @@ class XmliterTestCase(unittest.TestCase): [["one"], ["two"]], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -162,6 +166,7 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath("id/text()").getall(), []) self.assertEqual(node.xpath("price/text()").getall(), []) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -190,6 +195,7 @@ class XmliterTestCase(unittest.TestCase): ["http://www.mydummycompany.com/images/item1.jpg"], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -214,6 +220,7 @@ class XmliterTestCase(unittest.TestCase): with self.assertRaises(StopIteration): next(my_iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -226,10 +233,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -244,12 +253,25 @@ class XmliterTestCase(unittest.TestCase): ) -class LxmlXmliterTestCase(XmliterTestCase): - xmliter = staticmethod(xmliter_lxml) +class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): + xmliter = staticmethod(xmliter) - @mark.xfail(reason="known bug of the current implementation") - def test_xmliter_namespaced_nodename(self): - super().test_xmliter_namespaced_nodename() + def test_deprecation(self): + body = b""" + + + + + """ + with pytest.warns( + ScrapyDeprecationWarning, + match="xmliter", + ): + next(self.xmliter(body, "product")) + + +class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): + xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): body = b""" @@ -333,7 +355,7 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - result = [row for row in csv] + result = list(csv) self.assertEqual( result, [ @@ -355,7 +377,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -372,7 +394,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual( - [row for row in csv1], + list(csv1), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -385,7 +407,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual( - [row for row in csv2], + list(csv2), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -400,7 +422,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, { @@ -419,7 +441,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -436,7 +458,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -453,7 +475,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -524,6 +546,6 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" + type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" ) self.assertEqual(a, b) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index eae744df5..a8d080822 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,18 +1,26 @@ +import json import logging +import re import sys import unittest +from io import StringIO +from typing import Any, Dict, Mapping, MutableMapping +from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, + SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, failure_to_exc_info, ) from scrapy.utils.test import get_crawler +from tests.spiders import LogSpider class FailureToExcInfoTest(unittest.TestCase): @@ -106,3 +114,180 @@ class StreamLoggerTest(unittest.TestCase): with LogCapture() as log: print("test log msg") log.check(("test", "ERROR", "test log msg")) + + +@pytest.mark.parametrize( + ("base_extra", "log_extra", "expected_extra"), + ( + ( + {"spider": "test"}, + {"extra": {"log_extra": "info"}}, + {"extra": {"log_extra": "info", "spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": None}, + {"extra": {"spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": {"spider": "test2"}}, + {"extra": {"spider": "test"}}, + ), + ), +) +def test_spider_logger_adapter_process( + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict +): + logger = logging.getLogger("test") + spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) + + log_message = "test_log_message" + result_message, result_kwargs = spider_logger_adapter.process( + log_message, log_extra + ) + + assert result_message == log_message + assert result_kwargs == expected_extra + + +class LoggingTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + self.spider.log_debug(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_info_logging(self): + log_message = "Bar message" + self.spider.log_info(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_warning_logging(self): + log_message = "Baz message" + self.spider.log_warning(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_error_logging(self): + log_message = "Foo bar message" + self.spider.log_error(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_critical_logging(self): + log_message = "Foo bar baz message" + self.spider.log_critical(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + +class LoggingWithExtraTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + formatter = logging.Formatter( + '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' + ) + handler.setFormatter(formatter) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + self.regex_pattern = re.compile(r"^]+>$") + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + extra = {"important_info": "foo"} + self.spider.log_debug(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "DEBUG" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_info_logging(self): + log_message = "Bar message" + extra = {"important_info": "bar"} + self.spider.log_info(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "INFO" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_warning_logging(self): + log_message = "Baz message" + extra = {"important_info": "baz"} + self.spider.log_warning(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "WARNING" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_error_logging(self): + log_message = "Foo bar message" + extra = {"important_info": "foo bar"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_critical_logging(self): + log_message = "Foo bar baz message" + extra = {"important_info": "foo bar baz"} + self.spider.log_critical(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "CRITICAL" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_overwrite_spider_extra(self): + log_message = "Foo message" + extra = {"important_info": "foo", "spider": "shouldn't change"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69793ee75..ee3314d8e 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -7,6 +7,8 @@ from unittest import mock from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, + build_from_crawler, + build_from_settings, create_instance, load_object, rel_has_nofollow, @@ -153,6 +155,78 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(TypeError): create_instance(m, settings, None) + def test_build_from_crawler(self): + settings = mock.MagicMock() + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_crawler(mock, settings, crawler): + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + if hasattr(mock, "from_settings"): + self.assertEqual(mock.from_settings.call_count, 0) + self.assertEqual(mock.call_count, 0) + elif hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, settings, crawler) + m.reset_mock() + + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with self.assertRaises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) + + def test_build_from_settings(self): + settings = mock.MagicMock() + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_settings(mock, settings): + build_from_settings(mock, settings, *args, **kwargs) + if hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_settings() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_settings"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_settings(m, settings) + m.reset_mock() + + # Check adoption of crawler settings + m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) + m.from_settings.return_value = None + with self.assertRaises(TypeError): + build_from_settings(m, settings, *args, **kwargs) + def test_set_environ(self): assert os.environ.get("some_test_environ") is None with set_environ(some_test_environ="test_value"): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 90bd350a5..3831f4c21 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -6,6 +6,7 @@ import unittest import warnings from pathlib import Path +from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings @@ -38,20 +39,6 @@ class ProjectUtilsTest(unittest.TestCase): self.assertEqual(abspath, data_path(abspath)) -@contextlib.contextmanager -def set_env(**update): - modified = set(update.keys()) & set(os.environ.keys()) - update_after = {k: os.environ[k] for k in modified} - remove_after = frozenset(k for k in update if k not in os.environ) - try: - os.environ.update(update) - yield - finally: - os.environ.update(update_after) - for k in remove_after: - os.environ.pop(k) - - class GetProjectSettingsTestCase(unittest.TestCase): def test_valid_envvar(self): value = "tests.test_cmdline.settings" @@ -60,7 +47,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("error") - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value @@ -69,7 +56,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { "SCRAPY_FOO": "bar", } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SCRAPY_FOO") is None @@ -80,7 +67,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): "SCRAPY_FOO": "bar", "SCRAPY_SETTINGS_MODULE": value, } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value assert settings.get("SCRAPY_FOO") is None diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 80d2e8da1..4c60deafe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, ["x", "y"])) a.y = 1 - # differente attributes + # different attributes self.assertFalse(equal_attributes(a, b, ["x", "y"])) # test callable @@ -239,8 +239,11 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) if platform.python_implementation() == "CPython": - # doesn't work on CPython: https://bugs.python.org/issue42785 - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951 + self.assertIn( + get_func_args(operator.itemgetter(2), stripself=True), + [[], ["args", "kwargs"]], + ) elif platform.python_implementation() == "PyPy": self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ["obj"] diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e6d1abe3f..633077eec 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,22 +2,15 @@ import json import unittest import warnings from hashlib import sha1 -from typing import Dict, Mapping, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary -import pytest -from w3lib.url import canonicalize_url - from scrapy.http import Request -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import to_bytes from scrapy.utils.request import ( - _deprecated_fingerprint_cache, _fingerprint_cache, - _request_fingerprint_as_bytes, fingerprint, request_authenticate, - request_fingerprint, request_httprepr, request_to_curl, ) @@ -233,168 +226,6 @@ class FingerprintTest(unittest.TestCase): self.assertEqual(actual, expected) -class RequestFingerprintTest(FingerprintTest): - function = staticmethod(request_fingerprint) - cache = _deprecated_fingerprint_cache - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( - ( - Request("http://example.org"), - "b2e5245ef826fd9576c93bd6e392fce3133fab62", - {}, - ), - ( - Request("https://example.org"), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org?a"), - "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", - {}, - ), - ( - Request("https://example.org?a=b"), - "42e5fe149b147476e3f67ad0670c57b4cc57856a", - {}, - ), - ( - Request("https://example.org?a=b&a"), - "d23a9787cb56c6375c2cae4453c5a8c634526942", - {}, - ), - ( - Request("https://example.org?a=b&a=c"), - "9a18a7a8552a9182b7f1e05d33876409e421e5c5", - {}, - ), - ( - Request("https://example.org", method="POST"), - "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", - {}, - ), - ( - Request("https://example.org", body=b"a"), - "4bb136e54e715a4ea7a9dd1101831765d33f2d60", - {}, - ), - ( - Request("https://example.org", method="POST", body=b"a"), - "6c6595374a304b293be762f7b7be3f54e9947c65", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "515b633cb3ca502a33a9d8c890e889ec1e425e65", - {"include_headers": ["A"]}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "505c96e7da675920dfef58725e8c957dfdb38f47", - {"keep_fragments": True}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", - {"include_headers": ["A"], "keep_fragments": True}, - ), - ( - Request("https://example.org/ab"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ( - Request("https://example.org/a", body=b"b"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ) - - def setUp(self) -> None: - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - def tearDown(self) -> None: - warnings.simplefilter("default", ScrapyDeprecationWarning) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - -class RequestFingerprintDeprecationTest(unittest.TestCase): - def test_deprecation_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com")) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertFalse(any("non-default" in message for message in messages)) - - def test_deprecation_non_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com"), keep_fragments=True) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertTrue(any("non-default" in message for message in messages)) - - -class RequestFingerprintAsBytesTest(FingerprintTest): - function = staticmethod(_request_fingerprint_as_bytes) - cache = _deprecated_fingerprint_cache - known_hashes = RequestFingerprintTest.known_hashes - - def test_caching(self): - r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual( - self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) - ) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - def test_hashes(self): - actual = [ - self.function(request, **kwargs) for request, _, kwargs in self.known_hashes - ] - expected = [ - bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes - ] - self.assertEqual(actual, expected) - - -_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() - - -def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): - if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) - cache = _fingerprint_cache_2_6.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) - if cache_key not in cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if include_headers: - for hdr in include_headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), @@ -424,94 +255,16 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class BackwardCompatibilityTestCase(unittest.TestCase): - def test_function_backward_compatibility(self): - include_headers_to_test = ( - None, - ["Accept-Language"], - ["accept-language", "sessionid"], - ["SESSIONID", "Accept-Language"], - ) - for request_object in REQUEST_OBJECTS_TO_TEST: - for include_headers in include_headers_to_test: - for keep_fragments in (False, True): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fp = request_fingerprint( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - old_fp = request_fingerprint_2_6( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - self.assertEqual(fp, old_fp) - - def test_component_backward_compatibility(self): - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - crawler = get_crawler(prevent_warnings=False) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - - def test_custom_component_backward_compatibility(self): - """Tests that the backward-compatible request fingerprinting class featured - in the documentation is indeed backward compatible and does not cause a - warning to be logged.""" - - class RequestFingerprinter: - cache = WeakKeyDictionary() - - def fingerprint(self, request): - if request not in self.cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url))) - fp.update(request.body or b"") - self.cache[request] = fp.digest() - return self.cache[request] - - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings() as logged_warnings: - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - } - crawler = get_crawler(settings_dict=settings) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - self.assertFalse(logged_warnings) - - class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(prevent_warnings=False) + crawler = get_crawler() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), + fingerprint(request), ) - self.assertTrue(logged_warnings) def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } @@ -522,14 +275,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): crawler.request_fingerprinter.fingerprint(request), fingerprint(request), ) - self.assertFalse(logged_warnings) - - def test_unknown_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", - } - with self.assertRaises(ValueError): - get_crawler(settings_dict=settings) + self.assertTrue(logged_warnings) class CustomRequestFingerprinterTestCase(unittest.TestCase): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 661fb47a3..7ad86127b 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,10 +1,14 @@ import unittest from pathlib import Path +from time import process_time from urllib.parse import urlparse +import pytest + from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( + _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, @@ -166,3 +170,76 @@ class ResponseUtilsTest(unittest.TestCase): assert open_in_browser( r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 0.02 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" + + response = HtmlResponse("https://example.com", body=body) + + start_time = process_time() + + open_in_browser(response, lambda url: True) + + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) + + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 0.02 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b"b", + b"ab", + ), + ( + b"ac", + b"ac", + ), + ( + b"acccd", + b"acd", + ), + ( + b"ad", + b"ad", + ), + ), +) +def test_remove_html_comments(input_body, output_body): + assert ( + _remove_html_comments(input_body) == output_body + ), f"{_remove_html_comments(input_body)=} == {output_body=}" diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 65b99e0c4..60232f10b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" - def test_send_catch_log(self): - return super().test_send_catch_log() - @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") - def test_send_catch_log(self): - return super().test_send_catch_log() - class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 460ae40c3..dd1d26448 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider + import tests.test_utils_spider # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index cbe80e157..fc42c0d2f 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") + context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0042fe8f0..cce119001 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,8 +2,10 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ + import shutil from pathlib import Path +from tempfile import mkdtemp import OpenSSL.SSL from twisted.internet import defer, reactor @@ -24,7 +26,7 @@ from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Headers, Request from scrapy.settings import Settings -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ( BrokenDownloadResource, @@ -274,8 +276,7 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -440,8 +441,7 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) @@ -470,8 +470,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayload(self): s = "0123456789" * 10 settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) return getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory @@ -482,8 +482,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): settings = Settings( {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"} ) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) d = getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing new file mode 100644 index 000000000..162e31d0c --- /dev/null +++ b/tests_typing/test_spiders.mypy-testing @@ -0,0 +1,68 @@ +from typing import Any + +import pytest + +from scrapy.http import HtmlResponse, Response +from scrapy.spiders import Spider + + +class SimpleSpider(Spider): + pass + + +class SameOverrideSpider(Spider): + def parse(self, response: Response, **kwargs: Any) -> Any: + pass + + +class NoKwargsSpider(Spider): + def parse(self, response: Response) -> Any: + pass + + +class SpecificKwargsSpider(Spider): + def parse(self, response: Response, page: int) -> Any: + pass + + +class NarrowOverrideSpider(Spider): + # without type: ignore this produces several note lines in addition to an error line, + # which is unsupported by pytest-mypy-testing + def parse(self, response: HtmlResponse, **kwargs: Any) -> Any: # type: ignore[override] + pass + + +@pytest.mark.mypy_testing +def test_spider_parse() -> None: + spider = Spider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_simple() -> None: + spider = SimpleSpider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_same() -> None: + spider = SameOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_no_kwargs() -> None: + spider = NoKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_specific_kwargs() -> None: + spider = SpecificKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_narrow() -> None: + spider = NarrowOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any diff --git a/tox.ini b/tox.ini index 932c0b805..cde4243f3 100644 --- a/tox.ini +++ b/tox.ini @@ -7,15 +7,28 @@ envlist = pre-commit,pylint,typing,py minversion = 1.7.0 +[test-requirements] +deps = + attrs + pexpect >= 4.8.0 + pyftpdlib >= 1.5.8 + pygments + pytest + pytest-cov==4.0.0 + pytest-xdist + sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 + testfixtures + pywin32; sys_platform == "win32" + [testenv] deps = - -rtests/requirements.txt + {[test-requirements]deps} + # mitmproxy does not support PyPy - # Python 3.9+ requires mitmproxy >= 5.3.0 - # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 - #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' + mitmproxy; implementation_name != 'pypy' + # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by + # mitmproxy. + werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -33,19 +46,28 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.10.0 + typing-extensions==4.11.0 types-attrs==19.1.0 - types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 - types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-lxml==2024.4.14 + types-Pillow==10.2.0.20240423 + types-Pygments==2.17.0.20240310 + types-pyOpenSSL==24.0.0.20240417 + types-setuptools==69.5.0.20240423 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +basepython = python3.8 +deps = + {[test-requirements]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.3 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = @@ -57,7 +79,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.0.1 + pylint==3.1.0 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -71,6 +93,7 @@ commands = twine check dist/* [pinned] +basepython = python3.8 deps = cryptography==36.0.0 cssselect==0.9.1 @@ -85,9 +108,9 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.4.1 - -rtests/requirements.txt + {[test-requirements]deps} - # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment setenv = _SCRAPY_PINNED=true @@ -97,7 +120,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -107,7 +130,7 @@ setenv = commands = {[pinned]commands} [testenv:windows-pinned] -basepython = python3 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -128,15 +151,28 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] + uvloop; platform_system != "Windows" + bpython # optional for shell wrapper tests + brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + ipython [testenv:extra-deps-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} boto3==1.20.0 google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 + brotlipy + uvloop==0.14.0; platform_system != "Windows" + bpython==0.7.1 + zstandard==0.1; implementation_name != 'pypy' + ipython==2.0.0 + brotli==0.5.2; implementation_name != 'pypy' + brotlicffi==0.8.0; implementation_name == 'pypy' install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -147,6 +183,7 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} @@ -159,12 +196,12 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = {[testenv:pypy3]basepython} +basepython = pypy3.8 deps = {[pinned]deps} PyPyDispatcher==2.1.0 commands = - pytest --durations=10 scrapy tests + pytest {posargs:--durations=10 scrapy tests} install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -212,7 +249,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} botocore==1.4.87