diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a00b7cfb3..f76bf783d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.9.0 +current_version = 2.11.0 commit = True tag = True tag_name = {new_version} diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..63cae77e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question / Help +about: Ask a question about Scrapy or ask for help with your Scrapy code. +--- + +Thanks for taking an interest in Scrapy! + +The Scrapy GitHub issue tracker is not meant for questions or help. Please ask +for help in the [Scrapy community resources](https://scrapy.org/community/) +instead. + +The GitHub issue tracker's purpose is to deal with bug reports and feature +requests for the project itself. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..d6fc0f6c5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,10 @@ name: Checks on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: checks: runs-on: ubuntu-latest @@ -8,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: pylint - python-version: 3.8 @@ -17,12 +21,12 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: twinecheck steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -38,5 +42,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..affaa32a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,14 +4,18 @@ on: tags: - '[0-9]+.[0-9]+.[0-9]+' +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 174d245ca..252176464 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,16 +1,20 @@ name: macOS on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: macos-11 strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 8fcf90a18..f50a4d104 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,6 +1,10 @@ name: Ubuntu on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: ubuntu-latest @@ -8,9 +12,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py @@ -20,30 +21,45 @@ jobs: - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio - python-version: pypy3.9 env: TOXENV: pypy3 + - python-version: pypy3.10 + env: + TOXENV: pypy3 # pinned deps - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3.7 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned + - python-version: 3.8.17 + env: + TOXENV: extra-deps-pinned + - python-version: 3.8.17 + env: + TOXENV: botocore-pinned - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: extra-deps + - python-version: "3.12" + env: + TOXENV: botocore steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -51,7 +67,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f60c48841..757d62285 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,6 +1,10 @@ name: Windows on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: windows-latest @@ -8,31 +12,27 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: windows-pinned - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" + - python-version: "3.11" + env: + TOXENV: py + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio -# no binary package for lxml for 3.11 yet -# - python-version: "3.11" -# env: -# TOXENV: py -# - python-version: "3.11" -# env: -# TOXENV: asyncio steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf8808f2..0cff5cc73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,20 +5,20 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 # 6.0.0 drops Python 3.7 support + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.3.0 + rev: 23.9.1 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.5 # 5.12 drops Python 3.7 support + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.13.0 + rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.3.0 + - black==23.9.1 diff --git a/MANIFEST.in b/MANIFEST.in index ae7db51fa..4920dc0c3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include NEWS include scrapy/VERSION include scrapy/mime.types +include scrapy/py.typed include codecov.yml include conftest.py diff --git a/README.rst b/README.rst index 970bf2c35..14adff648 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,10 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS +.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + .. :alt: macOS + .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows @@ -41,7 +42,7 @@ Scrapy Overview ======== -Scrapy is a fast high-level web crawling and web scraping framework, used to +Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. @@ -58,7 +59,7 @@ including a list of features. Requirements ============ -* Python 3.7+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install @@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ for details. +See https://scrapy.org/support/ for details. \ No newline at end of file diff --git a/conftest.py b/conftest.py index e1d4b1213..2bfa46f5a 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,10 @@ +import platform +import sys from pathlib import Path import pytest +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -14,6 +18,10 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + "tests/ftpserver.py", + "tests/mockserver.py", + "tests/pipelines.py", + "tests/spiders.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess @@ -73,6 +81,20 @@ def only_not_asyncio(request, reactor_pytest): pytest.skip("This test is only run without --reactor=asyncio") +@pytest.fixture(autouse=True) +def requires_uvloop(request): + if not request.node.get_closest_marker("requires_uvloop"): + return + if sys.implementation.name == "pypy": + pytest.skip("uvloop does not support pypy properly") + if platform.system() == "Windows": + pytest.skip("uvloop does not support Windows") + if twisted_version == Version("twisted", 21, 2, 0): + pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + if sys.version_info >= (3, 12): + pytest.skip("uvloop doesn't support Python 3.12 yet") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..9ca0f817a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "coverage": ("https://coverage.readthedocs.io/en/latest", None), "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), diff --git a/docs/contributing.rst b/docs/contributing.rst index eef92e148..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.7 use:: +the tests with Python 3.10 use:: - tox -e py37 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py37,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.7 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py37 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -322,4 +322,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/index.rst b/docs/index.rst index 5404969e0..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -222,6 +222,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -235,6 +236,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 2c2079f68..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.7+, either the CPython implementation (default) or +Scrapy requires Python 3.8+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 19a76fc16..8ea98f29b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -493,7 +493,15 @@ in the callback, as you can see below: "tags": quote.css("div.tags a.tag::text").getall(), } -If you run this spider, it will output the extracted data with the log:: +To run this spider, exit the scrapy shell by entering:: + + quit() + +Then, run:: + + scrapy crawl quotes + +Now, it should output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} diff --git a/docs/news.rst b/docs/news.rst index c7ad11862..65d9c5181 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,307 @@ Release notes ============= +.. _release-2.11.0: + +Scrapy 2.11.0 (2023-09-18) +-------------------------- + +Highlights: + +- Spiders can now modify :ref:`settings ` in their + :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider + arguments `. + +- Periodic logging of stats. + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Most of the initialization of :class:`scrapy.crawler.Crawler` instances is + now done in :meth:`~scrapy.crawler.Crawler.crawl`, so the state of + instances before that method is called is now different compared to older + Scrapy versions. We do not recommend using the + :class:`~scrapy.crawler.Crawler` instances before + :meth:`~scrapy.crawler.Crawler.crawl` is called. (:issue:`6038`) + +- :meth:`scrapy.Spider.from_crawler` is now called before the initialization + of various components previously initialized in + :meth:`scrapy.crawler.Crawler.__init__` and before the settings are + finalized and frozen. This change was needed to allow changing the settings + in :meth:`scrapy.Spider.from_crawler`. If you want to access the final + setting values and the initialized :class:`~scrapy.crawler.Crawler` + attributes in the spider code as early as possible you can do this in + :meth:`~scrapy.Spider.start_requests` or in a handler of the + :signal:`engine_started` signal. (:issue:`6038`) + +- The :meth:`TextResponse.json ` method now + requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or + UTF-32). If you need to deal with JSON documents in an invalid encoding, + use ``json.loads(response.text)`` instead. (:issue:`6016`) + +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the binary export mode of + :class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0. + (:issue:`6006`, :issue:`6007`) + + .. note:: If you are using this Scrapy version on Scrapy Cloud with a stack + that includes an older Scrapy version and get a "TypeError: + Unexpected options: binary" error, you may need to add + ``scrapinghub-entrypoint-scrapy >= 0.14.1`` to your project + requirements or switch to a stack that includes Scrapy 2.11. + +- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy + 1.0.0, use :attr:`CrawlerRunner.spider_loader + ` instead. (:issue:`6010`) + +Deprecations +~~~~~~~~~~~~ + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`scrapy.crawler.Crawler` instance is now deprecated. (:issue:`1587`, + :issue:`6040`) + +New features +~~~~~~~~~~~~ + +- Spiders can now modify settings in their + :meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider + arguments `. (:issue:`1305`, :issue:`1580`, :issue:`2392`, + :issue:`3663`, :issue:`6038`) + +- Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension + which can be enabled to log stats and/or their differences periodically. + (:issue:`5926`) + +- Optimized the memory usage in :meth:`TextResponse.json + ` by removing unnecessary body decoding. + (:issue:`5968`, :issue:`6016`) + +- Links to ``.webp`` files are now ignored by :ref:`link extractors + `. (:issue:`6021`) + +Bug fixes +~~~~~~~~~ + +- Fixed logging enabled add-ons. (:issue:`6036`) + +- Fixed :class:`~scrapy.mail.MailSender` producing invalid message bodies + when the ``charset`` argument is passed to + :meth:`~scrapy.mail.MailSender.send`. (:issue:`5096`, :issue:`5118`) + +- Fixed an exception when accessing ``self.EXCEPTIONS_TO_RETRY`` from a + subclass of :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`6049`, :issue:`6050`) + +- :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse + :setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`, + :issue:`6013`) + +- Calls to ``datetime.utcnow()``, no longer recommended to be used, have been + replaced with calls to ``datetime.now()`` with a timezone. (:issue:`6014`) + +Documentation +~~~~~~~~~~~~~ + +- Updated a deprecated function call in a pipeline example. (:issue:`6008`, + :issue:`6009`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`6003`, :issue:`6005`, :issue:`6031`, + :issue:`6034`) + +- Pinned brotli_ to 1.0.9 for the PyPy tests as 1.1.0 breaks them. + (:issue:`6044`, :issue:`6045`) + +- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`, + :issue:`6046`) + +.. _release-2.10.1: + +Scrapy 2.10.1 (2023-08-30) +-------------------------- + +Marked ``Twisted >= 23.8.0`` as unsupported. (:issue:`6024`, :issue:`6026`) + +.. _release-2.10.0: + +Scrapy 2.10.0 (2023-08-04) +-------------------------- + +Highlights: + +- Added Python 3.12 support, dropped Python 3.7 support. + +- The new add-ons framework simplifies configuring 3rd-party components that + support it. + +- Exceptions to retry can now be configured. + +- Many fixes and improvements for feed exports. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.7. (:issue:`5953`) + +- Added support for the upcoming Python 3.12. (:issue:`5984`) + +- Minimum versions increased for these dependencies: + + - lxml_: 4.3.0 → 4.4.1 + + - cryptography_: 3.4.6 → 36.0.0 + +- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) + +- boto3_ is now recommended instead of botocore_ for exporting to S3. + (:issue:`5833`). + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, + :issue:`5847`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter, deprecated + in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed. + (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``, + ``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``, + and ``MarshalLifoDiskQueueNonRequest``, deprecated in + Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`) + +- The property ``open_spiders`` and the methods ``has_capacity`` and + ``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`, + deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`) + +- Passing a ``spider`` argument to the + :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`, + :meth:`~scrapy.core.engine.ExecutionEngine.crawl` and + :meth:`~scrapy.core.engine.ExecutionEngine.download` methods of + :class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is + no longer supported. (:issue:`5994`, :issue:`5998`) + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use + :class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead. + (:issue:`5146`) + +- Passing the ``custom`` argument to + :func:`scrapy.utils.conf.build_component_list` is deprecated, it was used + in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy + uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same. + Code that uses this argument and cannot be switched to ``getwithbase()`` + can be switched to merging the values explicitly. (:issue:`5726`, + :issue:`5923`) + +New features +~~~~~~~~~~~~ + +- Added support for :ref:`Scrapy add-ons `. (:issue:`5950`) + +- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which + exceptions will be retried by + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`2701`, :issue:`5929`) + +- Added the possiiblity to close the spider if no items were produced in the + specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + (:issue:`5979`) + +- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports. + (:issue:`5980`) + +- Added support for using :class:`pathlib.Path` objects that refer to + absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`) + +Bug fixes +~~~~~~~~~ + +- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``. + (:issue:`872`, :issue:`5847`) + +- Fixed using absolute Windows paths when specifying output files. + (:issue:`5969`, :issue:`5971`) + +- Fixed problems with uploading large files to S3 by switching to multipart + uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`) + +- Fixed the JSON exporter writing extra commas when some exceptions occur. + (:issue:`3090`, :issue:`5952`) + +- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`, + :issue:`5705`) + +- Fixed an error when a component added by the class object throws + :exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`, + :issue:`5992`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method. + (:issue:`5959`, :issue:`5960`, :issue:`5963`) + +- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement + for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API + inconsistencies. (:issue:`5146`) + +Documentation +~~~~~~~~~~~~~ + +- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`, + :issue:`5846`) + +- Documented possible problems with early Twisted reactor installation and + their solutions. (:issue:`5981`, :issue:`6000`) + +- Added examples of making additional requests in callbacks. (:issue:`5927`) + +- Improved the feed export docs. (:issue:`5579`, :issue:`5931`) + +- Clarified the docs about request objects on redirection. (:issue:`5707`, + :issue:`5937`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added support for running tests against the installed Scrapy version. + (:issue:`4914`, :issue:`5949`) + +- Extended typing hints. (:issue:`5925`, :issue:`5977`) + +- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop`` + test. (:issue:`5951`) + +- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ`` + test on Windows. (:issue:`5847`) + +- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`) + +- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`) + +- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`) + +- Implemented cleanups. (:issue:`5965`, :issue:`5986`) + .. _release-2.9.0: Scrapy 2.9.0 (2023-05-08) @@ -25,7 +326,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Settings correponding to :setting:`DOWNLOAD_DELAY`, +- Settings corresponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) @@ -173,6 +474,10 @@ Deprecation removals has now been removed. (:issue:`5719`) +- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + Deprecations ~~~~~~~~~~~~ @@ -5748,6 +6053,7 @@ First release of Scrapy. .. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..1bf2172bd --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,193 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. + +The ``ADDONS`` setting is a dict in which every key is an add-on class or its +import path and the value is its priority. + +This is an example where two add-ons are enabled in a project's +``settings.py``:: + + ADDONS = { + 'path.to.someaddon': 0, + SomeAddonClass: 1, + } + + +Writing your own add-ons +======================== + +Add-ons are Python classes that include the following method: + +.. method:: update_settings(settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an add-on instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access + them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this add-on + :type crawler: :class:`~scrapy.crawler.Crawler` + +The settings set by the add-on should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an add-on-specific setting that governs whether the add-on will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. + +Fallbacks +--------- + +Some components provided by add-ons need to fall back to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The add-ons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided by the add-on (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several add-ons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. + + +Add-on examples +=============== + +Set some basic configuration: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings.set("DNSCACHE_ENABLED", True, "addon") + +Check dependencies: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + try: + import boto + except ImportError: + raise NotConfigured("MyAddon requires the boto library") + ... + +Access the crawler instance: + +.. code-block:: python + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def update_settings(self, settings): + ... + +Use a fallback component: + +.. code-block:: python + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(FALLBACK_SETTING)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 268344879..175c877de 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. + motion. Should be called only once. Returns a deferred that is fired when the crawl is finished. @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..07baea071 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -27,54 +27,43 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') -.. _using-custom-loops: +.. _asyncio-preinstalled-reactor: -Using custom asyncio loops -========================== +Handling a pre-installed reactor +================================ -You can also use custom asyncio event loops with the asyncio reactor. Set the -:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to -use it instead of the default asyncio event loop. +``twisted.internet.reactor`` and some other Twisted imports install the default +Twisted reactor as a side effect. Once a Twisted reactor is installed, it is +not possible to switch to a different reactor at run time. + +If you :ref:`configure the asyncio Twisted reactor ` and, at +run time, Scrapy complains that a different reactor is already installed, +chances are you have some such imports in your code. + +You can usually fix the issue by moving those offending module-level Twisted +imports to the method or function definitions where they are used. For example, +if you have something like: + +.. code-block:: python + + from twisted.internet import reactor -.. _asyncio-windows: + def my_function(): + reactor.callLater(...) -Windows-specific notes -====================== +Switch to something like: -The Windows implementation of :mod:`asyncio` can use two event loop -implementations: +.. code-block:: python -- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required - when using Twisted. + def my_function(): + from twisted.internet import reactor -- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work - with Twisted. + reactor.callLater(...) -So on Python 3.8+ the event loop class needs to be changed. - -.. versionchanged:: 2.6.0 - The event loop class is changed automatically when you change the - :setting:`TWISTED_REACTOR` setting or call - :func:`~scrapy.utils.reactor.install_reactor`. - -To change the event loop class manually, call the following code before -installing the reactor:: - - import asyncio - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -You can put this in the same function that installs the reactor, if you do that -yourself, or in some code that runs before the reactor is installed, e.g. -``settings.py``. - -.. note:: Other libraries you use may require - :class:`~asyncio.ProactorEventLoop`, e.g. because it supports - subprocesses (this is the case with `playwright`_), so you cannot use - them together with Scrapy on Windows (but you should be able to use - them on WSL or native Linux). - -.. _playwright: https://github.com/microsoft/playwright-python +Alternatively, you can try to :ref:`manually install the asyncio reactor +`, with :func:`~scrapy.utils.reactor.install_reactor`, before +those imports happen. .. _asyncio-await-dfd: @@ -122,3 +111,36 @@ example: f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) + + +.. _asyncio-windows: + +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations, :class:`~asyncio.ProactorEventLoop` (default) and +:class:`~asyncio.SelectorEventLoop`. However, only +:class:`~asyncio.SelectorEventLoop` works with Twisted. + +Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event +loop class to use it instead of the default asyncio event loop. diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1ed55f000..478dd9647 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use .. code-block:: python - from pkg_resources import parse_version + from packaging.version import parse as parse_version import scrapy diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 7665a901a..1abbc4968 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -915,6 +915,7 @@ settings (see the settings documentation for more info): * :setting:`RETRY_ENABLED` * :setting:`RETRY_TIMES` * :setting:`RETRY_HTTP_CODES` +* :setting:`RETRY_EXCEPTIONS` .. reqmeta:: dont_retry @@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_EXCEPTIONS + +RETRY_EXCEPTIONS +^^^^^^^^^^^^^^^^ + +Default:: + + [ + 'twisted.internet.defer.TimeoutError', + 'twisted.internet.error.TimeoutError', + 'twisted.internet.error.DNSLookupError', + 'twisted.internet.error.ConnectionRefusedError', + 'twisted.internet.error.ConnectionDone', + 'twisted.internet.error.ConnectError', + 'twisted.internet.error.ConnectionLost', + 'twisted.internet.error.TCPTimedOutError', + 'twisted.web.client.ResponseFailed', + IOError, + 'scrapy.core.downloader.handlers.http11.TunnelError', + ] + +List of exceptions to retry. + +Each list entry may be an exception type or its import path as a string. + +An exception will not be caught when the exception type is not in +:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request +has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught +exception propagation, see +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1007,8 +1039,8 @@ RobotsTxtMiddleware * :ref:`Protego ` (default) * :ref:`RobotFileParser ` - * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` + * :ref:`Reppy ` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1101,6 +1133,7 @@ In order to use this parser: .. warning:: `Upstream issue #122 `_ prevents reppy usage in Python 3.9+. + Because of this the Reppy parser is deprecated. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 96e0216b8..f7b2f3799 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following settings: * :setting:`CLOSESPIDER_TIMEOUT` +* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` @@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by timeout. +.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM + +CLOSESPIDER_TIMEOUT_NO_ITEM +""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies a number of seconds. If the spider has not produced +any items in the last number of seconds, it will be closed with the reason +``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed +regardless if it hasn't produced any items. + .. setting:: CLOSESPIDER_ITEMCOUNT CLOSESPIDER_ITEMCOUNT @@ -337,6 +350,122 @@ full list of parameters, including examples on how to instantiate .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy +.. module:: scrapy.extensions.periodic_log + :synopsis: Periodic stats logging + +Periodic log extension +~~~~~~~~~~~~~~~~~~~~~~ + +.. class:: PeriodicLog + +This extension periodically logs rich stat data as a JSON object:: + + 2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min) + 2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: { + "delta": { + "downloader/request_bytes": 55582, + "downloader/request_count": 162, + "downloader/request_method_count/GET": 162, + "downloader/response_bytes": 618133, + "downloader/response_count": 162, + "downloader/response_status_count/200": 162, + "item_scraped_count": 161 + }, + "stats": { + "downloader/request_bytes": 338243, + "downloader/request_count": 992, + "downloader/request_method_count/GET": 992, + "downloader/response_bytes": 3836736, + "downloader/response_count": 976, + "downloader/response_status_count/200": 976, + "item_scraped_count": 925, + "log_count/INFO": 21, + "log_count/WARNING": 1, + "scheduler/dequeued": 992, + "scheduler/dequeued/memory": 992, + "scheduler/enqueued": 1050, + "scheduler/enqueued/memory": 1050 + }, + "time": { + "elapsed": 360.008903, + "log_interval": 60.0, + "log_interval_real": 60.006694, + "start_time": "2023-08-03 23:24:57", + "utcnow": "2023-08-03 23:30:57" + } + } + +This extension logs the following configurable sections: + +- ``"delta"`` shows how some numeric stats have changed since the last stats + log message. + + The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They + must have ``int`` or ``float`` values. + +- ``"stats"`` shows the current value of some stats. + + The :setting:`PERIODIC_LOG_STATS` setting determines the target stats. + +- ``"time"`` shows detailed timing data. + + The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or + not to show this section. + +This extension logs data at the start, then on a fixed time interval +configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally +right before the crawl ends. + + +Example extension configuration: + +.. code-block:: python + + custom_settings = { + "LOG_LEVEL": "INFO", + "PERIODIC_LOG_STATS": { + "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"], + }, + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "EXTENSIONS": { + "scrapy.extensions.periodic_log.PeriodicLog": 0, + }, + } + +.. setting:: PERIODIC_LOG_DELTA + +PERIODIC_LOG_DELTA +"""""""""""""""""" + +Default: ``None`` + +* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stat values. +* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - show deltas for stats with names containing any configured substring. +* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - show deltas for all stats with names not containing any configured substring. + +.. setting:: PERIODIC_LOG_STATS + +PERIODIC_LOG_STATS +"""""""""""""""""" + +Default: ``None`` + +* ``"PERIODIC_LOG_STATS": True`` - show the current value of all stats. +* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - show current values for stats with names containing any configured substring. +* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - show current values for all stats with names not containing any configured substring. + + +.. setting:: PERIODIC_LOG_TIMING_ENABLED + +PERIODIC_LOG_TIMING_ENABLED +""""""""""""""""""""""""""" + +Default: ``False`` + +``True`` enables logging of timing data (i.e. the ``"time"`` section). + + Debugging extensions -------------------- diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b4ac93b1d..700775e4b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -101,12 +101,12 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` - :ref:`topics-feed-storage-ftp` -- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-s3` (requires boto3_) - :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the botocore_ +not available. For example, the S3 backend is only available if the boto3_ library is installed. @@ -156,8 +156,8 @@ The feeds are stored in the local filesystem. - Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if -you specify an absolute path like ``/tmp/export.csv``. This only works on Unix -systems though. +you specify an absolute path like ``/tmp/export.csv`` (Unix systems only). +Alternatively you can also use a :class:`pathlib.Path` object. .. _topics-feed-storage-ftp: @@ -199,7 +199,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `botocore`_ >= 1.4.87 +- Required external libraries: `boto3`_ >= 1.20.0 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -210,10 +210,12 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys -You can also define a custom ACL and custom endpoint for exported feeds using this setting: +You can also define a custom ACL, custom endpoint, and region name for exported +feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +- :setting:`AWS_REGION_NAME` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. @@ -572,9 +574,12 @@ to ``.json`` or ``.xml``. FEED_STORE_EMPTY ---------------- -Default: ``False`` +Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option +` is enabled. .. setting:: FEED_STORAGES @@ -799,6 +804,6 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ -.. _botocore: https://github.com/boto/botocore +.. _boto3: https://github.com/boto/boto3 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index bc26bbebe..a5f6e07b8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -215,7 +215,7 @@ item. screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) response = await maybe_deferred_to_future( - spider.crawler.engine.download(request, spider) + spider.crawler.engine.download(request) ) if response.status != 200: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index f64da22d8..b1b8c9e9c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ +* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy + plugin `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders .. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ +.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 407df32d2..8edf710bc 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,20 +115,9 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - To create a request that does not send stored cookies and does not - store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. - - Example of a request that sends manually-defined cookies and ignores - cookie storage: - - .. code-block:: python - - Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - meta={"dont_merge_cookies": True}, - ) + Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in + :attr:`request.meta ` causes custom cookies to be + ignored. For more info see :ref:`cookies-mw`. @@ -204,18 +193,47 @@ Request objects :meth:`replace`. .. attribute:: Request.meta + :value: {} - A dict that contains arbitrary metadata for this request. This dict is - empty for new Requests, and is usually populated by different Scrapy - components (extensions, middlewares, etc). So the data contained in this - dict depends on the extensions you have enabled. + A dictionary of arbitrary metadata for the request. - See :ref:`topics-request-meta` for a list of special meta keys - recognized by Scrapy. + You may extend request metadata as you see fit. - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.meta`` attribute. + Request metadata can also be accessed through the + :attr:`~scrapy.http.Response.meta` attribute of a response. + + To pass data from one spider callback to another, consider using + :attr:`cb_kwargs` instead. However, request metadata may be the right + choice in certain scenarios, such as to maintain some debugging data + across all follow-up requests (e.g. the source URL). + + A common use of request metadata is to define request-specific + parameters for Scrapy components (extensions, middlewares, etc.). For + example, if you set ``dont_retry`` to ``True``, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never + retry that request, even if it fails. See :ref:`topics-request-meta`. + + You may also use request metadata in your custom Scrapy components, for + example, to keep request state information relevant to your component. + For example, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the + ``retry_times`` metadata key to keep track of how many times a request + has been retried so far. + + Copying all the metadata of a previous request into a new, follow-up + request in a spider callback is a bad practice, because request + metadata may include metadata set by Scrapy components that is not + meant to be copied into other requests. For example, copying the + ``retry_times`` metadata key into follow-up requests can lower the + amount of retries allowed for those follow-up requests. + + You should only copy all request metadata from one request to another + if the new request is meant to replace the old request, as is often the + case when returning a request from a :ref:`downloader middleware + ` method. + + Also mind that the :meth:`copy` and :meth:`replace` request methods + :doc:`shallow-copy ` request metadata. .. attribute:: Request.cb_kwargs @@ -1103,9 +1121,10 @@ Response objects through all :ref:`Downloader Middlewares `. In particular, this means that: - - HTTP redirections will cause the original request (to the URL before - redirection) to be assigned to the redirected response (with the final - URL after redirection). + - HTTP redirections will create a new request from the request before + redirection. It has the majority of the same metadata and original + request attributes and gets assigned to the redirected response + instead of the propagation of the original request. - Response.request.url doesn't always equal Response.url diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3e06d84f9..06f8481ba 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by add-ons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -66,8 +67,8 @@ Example:: ---------------------- Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +own settings that will take precedence and override the project ones. One way +to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: .. code-block:: python @@ -81,6 +82,45 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } +It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, +and settings set there should use the "spider" priority explicitly: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def update_settings(cls, settings): + super().update_settings(settings) + settings.set("SOME_SETTING", "some value", priority="spider") + +.. versionadded:: 2.11 + +It's also possible to modify the settings in the +:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider +arguments ` or other logic: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + if "some_argument" in kwargs: + spider.settings.set( + "SOME_SETTING", kwargs["some_argument"], priority="spider" + ) + return spider + 3. Project settings module -------------------------- @@ -89,7 +129,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by add-ons +-------------------------- + +:ref:`Add-ons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +143,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` @@ -201,6 +247,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``{}`` + +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -964,7 +1020,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR @@ -1065,7 +1120,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. JOBDIR ------ -Default: ``''`` +Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 788bd7678..30677fe74 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -136,6 +136,21 @@ scrapy.Spider attributes in the new instance so they can be accessed later inside the spider's code. + .. versionchanged:: 2.11 + + The settings in ``crawler.settings`` can now be modified in this + method, which is handy if you want to modify them based on + arguments. As a consequence, these settings aren't the final values + as they can be modified later by e.g. :ref:`add-ons + `. For the same reason, most of the + :class:`~scrapy.crawler.Crawler` attributes aren't initialized at + this point. + + The final settings and the initialized + :class:`~scrapy.crawler.Crawler` attributes are available in the + :meth:`start_requests` method, handlers of the + :signal:`engine_started` signal and later. + :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance @@ -145,6 +160,46 @@ scrapy.Spider :param kwargs: keyword arguments passed to the :meth:`__init__` method :type kwargs: dict + .. classmethod:: update_settings(settings) + + The ``update_settings()`` method is used to modify the spider's settings + and is called during initialization of a spider instance. + + It takes a :class:`~scrapy.settings.Settings` object as a parameter and + can add or update the spider's configuration values. This method is a + class method, meaning that it is called on the :class:`~scrapy.Spider` + class and allows all instances of the spider to share the same + configuration. + + While per-spider settings can be set in + :attr:`~scrapy.Spider.custom_settings`, using ``update_settings()`` + allows you to dynamically add, remove or change settings based on other + settings, spider attributes or other factors and use setting priorities + other than ``'spider'``. Also, it's easy to extend ``update_settings()`` + in a subclass by overriding it, while doing the same with + :attr:`~scrapy.Spider.custom_settings` can be hard. + + For example, suppose a spider needs to modify :setting:`FEEDS`: + + .. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + custom_feed = { + "/home/user/documents/items.json": { + "format": "json", + "indent": 4, + } + } + + @classmethod + def update_settings(cls, settings): + super().update_settings(settings) + settings.setdefault("FEEDS", {}).update(cls.custom_feed) + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 1f270837c..c17b9d511 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -30,7 +30,7 @@ def main(): try: with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() - except IOError: + except OSError: print("linkcheck output not found; please run linkcheck first.") sys.exit(1) diff --git a/pytest.ini b/pytest.ini index 866f0c950..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed + requires_uvloop: marks tests as only enabled when uvloop is known to be working filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/scrapy/VERSION b/scrapy/VERSION index c8e38b614..46b81d815 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.9.0 +2.11.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a757a9290..cc0e539c4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 7): - print(f"Scrapy {__version__} requires Python 3.7+") +if sys.version_info < (3, 8): + print(f"Scrapy {__version__} requires Python 3.8+") sys.exit(1) diff --git a/scrapy/addons.py b/scrapy/addons.py new file mode 100644 index 000000000..9060d4f3f --- /dev/null +++ b/scrapy/addons.py @@ -0,0 +1,53 @@ +import logging +from typing import TYPE_CHECKING, Any, List + +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings +from scrapy.utils.conf import build_component_list +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + +logger = logging.getLogger(__name__) + + +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" + + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler + self.addons: List[Any] = [] + + def load_settings(self, settings: Settings) -> None: + """Load add-ons and configurations from a settings object and apply them. + + This will load the add-on for every add-on path in the + ``ADDONS`` setting and execute their ``update_settings`` methods. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + logger.warning( + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": self.addons, + }, + extra={"crawler": self.crawler}, + ) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 730e55350..6580ba9ce 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,8 +3,7 @@ import cProfile import inspect import os import sys - -import pkg_resources +from importlib.metadata import entry_points import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -49,7 +48,11 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in pkg_resources.iter_entry_points(group): + if sys.version_info >= (3, 10): + eps = entry_points(group=group) + else: + eps = entry_points().get(group, ()) + for entry_point in eps: obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9baee3a48..2aa569cdd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,7 +4,7 @@ Base class for Scrapy commands import argparse import os from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from twisted.python import failure @@ -116,7 +116,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1359e445f..cdb7ad4ae 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,10 @@ import sys +from argparse import Namespace +from typing import List, Type from w3lib.url import is_url +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request @@ -57,7 +60,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_): sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -73,7 +76,8 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: spidercls = spider_loader.load(opts.spider) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 63c23d04c..12e37babc 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,8 +3,11 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +from argparse import Namespace from threading import Thread +from typing import List, Type +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -54,15 +57,16 @@ class Command(ScrapyCommand): """ pass - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file url = guess_scheme(url) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: @@ -73,6 +77,7 @@ class Command(ScrapyCommand): # The crawler is created this way since the Shell manually handles the # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) + crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() crawler.engine.start() diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 86098edca..d46eb7c51 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,7 +2,8 @@ import re import sys from functools import wraps from inspect import getmembers -from typing import Dict +from types import CoroutineType +from typing import AsyncGenerator, Dict, Optional, Type from unittest import TestCase from scrapy.http import Request @@ -13,7 +14,7 @@ from scrapy.utils.spider import iterate_spider_output class Contract: """Abstract class for contracts""" - request_cls = None + request_cls: Optional[Type[Request]] = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") @@ -37,7 +38,12 @@ class Contract: else: results.addSuccess(self.testcase_pre) finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list( # pylint: disable=return-in-finally + iterate_spider_output(cb_result) + ) request.callback = wrapper @@ -49,7 +55,10 @@ class Contract: @wraps(cb) def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + output = list(iterate_spider_output(cb_result)) try: results.startTest(self.testcase_post) self.post_process(output) @@ -61,7 +70,7 @@ class Contract: else: results.addSuccess(self.testcase_post) finally: - return output + return output # pylint: disable=return-in-finally request.callback = wrapper diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 426056dc8..c84525160 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -154,7 +154,7 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred = Deferred().addBoth(_deactivate) + deferred: Deferred = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index 8b78c53c1..a7ae56a85 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,9 @@ +from typing import Any, Dict + from w3lib.url import parse_data_uri -from scrapy.http import TextResponse +from scrapy import Request, Spider +from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -9,11 +12,11 @@ class DataURIDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs = {} + resp_kwargs: Dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") resp_kwargs["encoding"] = charset diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3e5a281b2..dd1f56f8c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst """ import logging -import warnings from time import time from typing import ( TYPE_CHECKING, @@ -14,7 +13,6 @@ from typing import ( Generator, Iterable, Iterator, - List, Optional, Set, Type, @@ -29,7 +27,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, DontCloseSpider from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -88,6 +86,7 @@ class ExecutionEngine: self.crawler: "Crawler" = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals + assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter self.slot: Optional[Slot] = None self.spider: Optional[Spider] = None @@ -213,7 +212,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request, self.spider) + d = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -222,7 +221,12 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type] + + def _remove_request(_: Any) -> None: + assert self.slot + self.slot.remove_request(request) + + d.addBoth(_remove_request) d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -266,13 +270,7 @@ class ExecutionEngine: ) return d - def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + def spider_is_idle(self) -> bool: if self.slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] @@ -285,18 +283,8 @@ class ExecutionEngine: return False return True - def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - raise RuntimeError( - f"The spider {spider.name!r} does not match the open spider" - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) @@ -311,39 +299,24 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + def download(self, request: Request) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - logger.warning( - "The spider '%s' does not match the open spider", spider.name - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request, spider).addBoth( - self._downloaded, request, spider - ) + return self._download(request).addBoth(self._downloaded, request) def _downloaded( - self, result: Union[Response, Request], request: Request, spider: Spider - ) -> Union[Deferred, Response]: + self, result: Union[Response, Request, Failure], request: Request + ) -> Union[Deferred, Response, Failure]: assert self.slot is not None # typing self.slot.remove_request(request) - return self.download(result, spider) if isinstance(result, Request) else result + return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: + def _download(self, request: Request) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) - if spider is None: - spider = self.spider - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError( @@ -352,15 +325,17 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - assert spider is not None - logkws = self.logformatter.crawled(result.request, result, spider) + assert self.spider is not None + logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.spider} + ) self.signals.send_catch_log( signal=signals.response_received, response=result, request=result.request, - spider=spider, + spider=self.spider, ) return result @@ -369,8 +344,8 @@ class ExecutionEngine: self.slot.nextcall.schedule() return _ - assert spider is not None - dwld = self.downloader.fetch(request, spider) + assert self.spider is not None + dwld = self.downloader.fetch(request, self.spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) return dwld @@ -394,6 +369,7 @@ class ExecutionEngine: if hasattr(scheduler, "open"): yield scheduler.open(spider) yield self.scraper.open_spider(spider) + assert self.crawler.stats self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) self.slot.nextcall.schedule() @@ -465,7 +441,11 @@ class ExecutionEngine: ) dfd.addErrback(log_failure("Error while sending spider_close signal")) - dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) + def close_stats(_: Any) -> None: + assert self.crawler.stats + self.crawler.stats.close_spider(spider, reason=reason) + + dfd.addBoth(close_stats) dfd.addErrback(log_failure("Stats close failure")) dfd.addBoth( @@ -485,31 +465,3 @@ class ExecutionEngine: dfd.addBoth(lambda _: self._spider_closed_callback(spider)) return dfd - - @property - def open_spiders(self) -> List[Spider]: - warnings.warn( - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return [self.spider] if self.spider is not None else [] - - def has_capacity(self) -> bool: - warnings.warn( - "ExecutionEngine.has_capacity is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return not bool(self.slot) - - def schedule(self, request: Request, spider: Spider) -> None: - warnings.warn( - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if self.slot is None: - raise RuntimeError("Engine slot not assigned") - self._schedule_request(request, spider) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 6c6ed6f9b..39d5921f4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -228,8 +228,8 @@ class Stream: content_length_name = self._request.headers.normkey(b"Content-Length") for name, values in self._request.headers.items(): - for value in values: - value = str(value, "utf-8") + for value_bytes in values: + value = str(value_bytes, "utf-8") if name == content_length_name: if value != content_length: logger.warning( diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3fb0bbaff..17c95f1ea 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import json import logging from abc import abstractmethod from pathlib import Path -from typing import Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred @@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. """ @@ -325,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.dqdir state = self._read_dqs_state(self.dqdir) q = create_instance( self.pqclass, @@ -344,7 +352,7 @@ class Scheduler(BaseScheduler): def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """Return a folder name to keep disk queue state at""" - if jobdir is not None: + if jobdir: dqdir = Path(jobdir, "requests.queue") if not dqdir.exists(): dqdir.mkdir(parents=True) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a85f6a661..8fb16b8a9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -110,6 +110,7 @@ class Scraper: self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler self.signals: SignalManager = crawler.signals + assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks @@ -205,6 +206,7 @@ class Scraper: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request + assert result.request callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) @@ -221,7 +223,11 @@ class Scraper: return dfd.addCallback(iterate_spider_output) def handle_spider_error( - self, _failure: Failure, request: Request, response: Response, spider: Spider + self, + _failure: Failure, + request: Request, + response: Union[Response, Failure], + spider: Spider, ) -> None: exc = _failure.value if isinstance(exc, CloseSpider): @@ -240,6 +246,7 @@ class Scraper: response=response, spider=spider, ) + assert self.crawler.stats self.crawler.stats.inc_value( f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) @@ -248,7 +255,7 @@ class Scraper: self, result: Union[Iterable, AsyncIterable], request: Request, - response: Response, + response: Union[Response, Failure], spider: Spider, ) -> Deferred: if not result: @@ -364,6 +371,7 @@ class Scraper: spider=spider, exception=output.value, ) + assert ex logkws = self.logformatter.item_error(item, ex, response, spider) logger.log( *logformatter_adapter(logkws), diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 256f6e2c5..1d3a11208 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,10 +4,14 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast -from twisted.internet import defer -from zope.interface.exceptions import DoesNotImplement +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) try: # zope >= 5.0 only supports MultipleInvalid @@ -18,12 +22,13 @@ except ImportError: from zope.interface.verify import verifyClass from scrapy import Spider, signals +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( @@ -54,7 +59,7 @@ class Crawler: def __init__( self, spidercls: Type[Spider], - settings: Union[None, dict, Settings] = None, + settings: Union[None, Dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -66,31 +71,45 @@ class Crawler: self.spidercls: Type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) + self._update_root_log_handler() + self.addons: AddonManager = AddonManager(self) self.signals: SignalManager = SignalManager(self) - self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) + self._init_reactor: bool = init_reactor + self.crawling: bool = False + self._started: bool = False - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - - d = dict(overridden_settings(self.settings)) - logger.info( - "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} - ) + self.extensions: Optional[ExtensionManager] = None + self.stats: Optional[StatsCollector] = None + self.logformatter: Optional[LogFormatter] = None + self.request_fingerprinter: Optional[RequestFingerprinter] = None + self.spider: Optional[Spider] = None + self.engine: Optional[ExecutionEngine] = None + def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) + + def _apply_settings(self) -> None: + if self.settings.frozen: + return + + self.addons.load_settings(self.settings) + self.stats = load_object(self.settings["STATS_CLASS"])(self) + + handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) + logging.root.addHandler(handler) # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving __init__ scope + # garbage collected after leaving the scope self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter: LogFormatter = lf_cls.from_crawler(self) + self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter: RequestFingerprinter = create_instance( + self.request_fingerprinter = create_instance( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), settings=self.settings, crawler=self, @@ -98,7 +117,7 @@ class Crawler: reactor_class: str = self.settings["TWISTED_REACTOR"] event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if init_reactor: + if self._init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor if reactor_class: @@ -111,44 +130,54 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) - self.extensions: ExtensionManager = ExtensionManager.from_crawler(self) - + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() - self.crawling: bool = False - self.spider: Optional[Spider] = None - self.engine: Optional[ExecutionEngine] = None - @defer.inlineCallbacks - def crawl(self, *args, **kwargs): + d = dict(overridden_settings(self.settings)) + logger.info( + "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} + ) + + @inlineCallbacks + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") - self.crawling = True + if self._started: + warnings.warn( + "Running Crawler.crawl() more than once is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawling = self._started = True try: self.spider = self._create_spider(*args, **kwargs) + self._apply_settings() + self._update_root_log_handler() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) - yield defer.maybeDeferred(self.engine.start) + yield maybeDeferred(self.engine.start) except Exception: self.crawling = False if self.engine is not None: yield self.engine.close() raise - def _create_spider(self, *args, **kwargs): + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) - def _create_engine(self): + def _create_engine(self) -> ExecutionEngine: return ExecutionEngine(self, lambda _: self.stop()) - @defer.inlineCallbacks - def stop(self): + @inlineCallbacks + def stop(self) -> Generator[Deferred, Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False - yield defer.maybeDeferred(self.engine.stop) + assert self.engine + yield maybeDeferred(self.engine.stop) class CrawlerRunner: @@ -171,45 +200,28 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings): + def _get_spider_loader(settings: BaseSettings): """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - excs = ( - (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement - ) - try: - verifyClass(ISpiderLoader, loader_cls) - except excs: - warnings.warn( - "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does " - "not fully implement scrapy.interfaces.ISpiderLoader interface. " - "Please add all missing methods to avoid unexpected runtime errors.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[Deferred] = set() self.bootstrap_failed = False - @property - def spiders(self): - warnings.warn( - "CrawlerRunner.spiders attribute is renamed to " - "CrawlerRunner.spider_loader.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return self.spider_loader - - def crawl(self, crawler_or_spidercls, *args, **kwargs): + def crawl( + self, + crawler_or_spidercls: Union[Type[Spider], str, Crawler], + *args: Any, + **kwargs: Any, + ) -> Deferred: """ Run a crawler with the provided arguments. @@ -239,12 +251,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler, *args, **kwargs): + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result): + def _done(result: Any) -> Any: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -252,7 +264,9 @@ class CrawlerRunner: return d.addBoth(_done) - def create_crawler(self, crawler_or_spidercls): + def create_crawler( + self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -272,21 +286,22 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + # temporary cast until self.spider_loader is typed + return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self): + def stop(self) -> Deferred: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in list(self.crawlers)]) + return DeferredList([c.stop() for c in list(self.crawlers)]) - @defer.inlineCallbacks - def join(self): + @inlineCallbacks + def join(self) -> Generator[Deferred, Any, None]: """ join() @@ -294,7 +309,7 @@ class CrawlerRunner: completed their executions. """ while self._active: - yield defer.DeferredList(self._active) + yield DeferredList(self._active) class CrawlerProcess(CrawlerRunner): @@ -321,13 +336,17 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True): + def __init__( + self, + settings: Union[Dict[str, Any], Settings, None] = None, + install_root_handler: bool = True, + ): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False - def _signal_shutdown(self, signum, _): + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) @@ -338,7 +357,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._graceful_stop_reactor) - def _signal_kill(self, signum, _): + def _signal_kill(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) @@ -348,14 +367,19 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - return Crawler(spidercls, self.settings, init_reactor=init_reactor) + # temporary cast until self.spider_loader is typed + return Crawler( + cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + ) - def start(self, stop_after_crawl=True, install_signal_handlers=True): + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -367,8 +391,8 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished - :param bool install_signal_handlers: whether to install the shutdown - handlers (default: True) + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) """ from twisted.internet import reactor @@ -379,22 +403,24 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - if install_signal_handlers: - install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) reactor.addSystemEventTrigger("before", "shutdown", self.stop) - reactor.run(installSignalHandlers=False) # blocking call + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self): + def _graceful_stop_reactor(self) -> Deferred: d = self.stop() d.addBoth(self._stop_reactor) return d - def _stop_reactor(self, _=None): + def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor try: diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 04ae719de..0e757e4be 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,10 +1,20 @@ +from __future__ import annotations + import logging import re +from typing import TYPE_CHECKING, Union from w3lib import html +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -15,7 +25,7 @@ class AjaxCrawlMiddleware: For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured @@ -23,13 +33,15 @@ class AjaxCrawlMiddleware: # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768) + self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if not isinstance(response, HtmlResponse) or response.status != 200: return response @@ -54,7 +66,7 @@ class AjaxCrawlMiddleware: ajax_crawl_request.meta["ajax_crawlable"] = True return ajax_crawl_request - def _has_ajax_crawlable_variant(self, response): + def _has_ajax_crawlable_variant(self, response: Response) -> bool: """ Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. @@ -64,12 +76,12 @@ class AjaxCrawlMiddleware: # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile( +_ajax_crawlable_re: re.Pattern[str] = re.compile( r'' ) -def _has_ajaxcrawlable_meta(text): +def _has_ajaxcrawlable_meta(text: str) -> bool: """ >>> _has_ajaxcrawlable_meta('') True diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6495157d7..85781efd6 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,21 +1,41 @@ +from __future__ import annotations + import logging from collections import defaultdict +from http.cookiejar import Cookie +from typing import ( + TYPE_CHECKING, + Any, + DefaultDict, + Dict, + Iterable, + Optional, + Sequence, + Union, +) from tldextract import TLDExtract +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) -def _is_public_domain(domain): +def _is_public_domain(domain: str) -> bool: parts = _split_domain(domain) return not parts.domain @@ -23,23 +43,27 @@ def _is_public_domain(domain): class CookiesMiddleware: """This middleware enables working with sites that need cookies""" - def __init__(self, debug=False): - self.jars = defaultdict(CookieJar) - self.debug = debug + def __init__(self, debug: bool = False): + self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.debug: bool = debug @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured return cls(crawler.settings.getbool("COOKIES_DEBUG")) - def _process_cookies(self, cookies, *, jar, request): + def _process_cookies( + self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request + ) -> None: for cookie in cookies: cookie_domain = cookie.domain if cookie_domain.startswith("."): cookie_domain = cookie_domain[1:] - request_domain = urlparse_cached(request).hostname.lower() + hostname = urlparse_cached(request).hostname + assert hostname is not None + request_domain = hostname.lower() if cookie_domain and _is_public_domain(cookie_domain): if cookie_domain != request_domain: @@ -48,9 +72,11 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_merge_cookies", False): - return + return None cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] @@ -61,8 +87,11 @@ class CookiesMiddleware: request.headers.pop("Cookie", None) jar.add_cookie_header(request) self._debug_cookie(request, spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_merge_cookies", False): return response @@ -76,7 +105,7 @@ class CookiesMiddleware: return response - def _debug_cookie(self, request, spider): + def _debug_cookie(self, request: Request, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -87,7 +116,7 @@ class CookiesMiddleware: msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _debug_set_cookie(self, response, spider): + def _debug_set_cookie(self, response: Response, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -98,7 +127,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie, request): + def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -109,7 +138,7 @@ class CookiesMiddleware: if key in ("name", "value"): msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) - return + return None continue if isinstance(cookie[key], (bool, float, int, str)): decoded[key] = str(cookie[key]) @@ -129,12 +158,15 @@ class CookiesMiddleware: cookie_str += f"; {key.capitalize()}={value}" return cookie_str - def _get_request_cookies(self, jar, request): + def _get_request_cookies( + self, jar: CookieJar, request: Request + ) -> Sequence[Cookie]: """ Extract cookies from the Request.cookies attribute """ if not request.cookies: return [] + cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py deleted file mode 100644 index 5839dc243..000000000 --- a/scrapy/downloadermiddlewares/decompression.py +++ /dev/null @@ -1,94 +0,0 @@ -""" This module implements the DecompressionMiddleware which tries to recognise -and extract the potentially compressed responses that may arrive. -""" - -import bz2 -import gzip -import logging -import tarfile -import zipfile -from io import BytesIO -from tempfile import mktemp -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.responsetypes import responsetypes - -warn( - "scrapy.downloadermiddlewares.decompression is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, -) - - -logger = logging.getLogger(__name__) - - -class DecompressionMiddleware: - """This middleware tries to recognise and extract the possibly compressed - responses that may arrive.""" - - def __init__(self): - self._formats = { - "tar": self._is_tar, - "zip": self._is_zip, - "gz": self._is_gzip, - "bz2": self._is_bzip2, - } - - def _is_tar(self, response): - archive = BytesIO(response.body) - try: - tar_file = tarfile.open(name=mktemp(), fileobj=archive) - except tarfile.ReadError: - return - - body = tar_file.extractfile(tar_file.members[0]).read() - respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body) - return response.replace(body=body, cls=respcls) - - def _is_zip(self, response): - archive = BytesIO(response.body) - try: - zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipFile: - return - - namelist = zip_file.namelist() - body = zip_file.read(namelist[0]) - respcls = responsetypes.from_args(filename=namelist[0], body=body) - return response.replace(body=body, cls=respcls) - - def _is_gzip(self, response): - archive = BytesIO(response.body) - try: - body = gzip.GzipFile(fileobj=archive).read() - except IOError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def _is_bzip2(self, response): - try: - body = bz2.decompress(response.body) - except IOError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def process_response(self, request, response, spider): - if not response.body: - return response - - for fmt, func in self._formats.items(): - new_response = func(response) - if new_response: - logger.debug( - "Decompressed response with format: %(responsefmt)s", - {"responsefmt": fmt}, - extra={"spider": spider}, - ) - return new_response - return response diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index cdacc7368..8aec37cf1 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,19 +3,32 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations +from typing import TYPE_CHECKING, Iterable, Tuple, Union + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class DefaultHeadersMiddleware: - def __init__(self, headers): - self._headers = headers + def __init__(self, headers: Iterable[Tuple[str, str]]): + self._headers: Iterable[Tuple[str, str]] = headers @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: for k, v in self._headers: request.headers.setdefault(k, v) + return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index a926ecf56..1c904c05b 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,23 +3,35 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations -from scrapy import signals +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class DownloadTimeoutMiddleware: - def __init__(self, timeout=180): - self._timeout = timeout + def __init__(self, timeout: float = 180): + self._timeout: float = timeout @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT")) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self._timeout = getattr(spider, "download_timeout", self._timeout) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) + return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index de5a81388..63490a37a 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,50 +3,45 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -import warnings + +from __future__ import annotations + +from typing import TYPE_CHECKING, Union from w3lib.http import basic_auth_header -from scrapy import signals -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.httpobj import urlparse_cached +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls() crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: usr = getattr(spider, "http_user", "") pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - if not hasattr(spider, "http_auth_domain"): - warnings.warn( - "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security " - "problems if the spider makes requests to several different domains. http_auth_domain " - "will be set to the domain of the first request, please set it to the correct value " - "explicitly.", - category=ScrapyDeprecationWarning, - ) - self.domain_unset = True - else: - self.domain = spider.http_auth_domain - self.domain_unset = False + self.domain = spider.http_auth_domain # type: ignore[attr-defined] - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: - domain = urlparse_cached(request).hostname - if self.domain_unset: - self.domain = domain - self.domain_unset = False if not self.domain or url_is_from_any_domain(request.url, [self.domain]): request.headers[b"Authorization"] = auth + return None diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index b9316c43a..971473403 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from email.utils import formatdate -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -23,7 +25,9 @@ from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object -HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class HttpCacheMiddleware: @@ -37,7 +41,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, + OSError, ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: @@ -49,9 +53,8 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler( - cls: Type[HttpCacheMiddlewareTV], crawler: Crawler - ) -> HttpCacheMiddlewareTV: + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -63,7 +66,9 @@ class HttpCacheMiddleware: def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request: Request, spider: Spider) -> Optional[Response]: + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_cache", False): return None @@ -73,7 +78,9 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse = self.storage.retrieve_response(spider, request) + cachedresponse: Optional[Response] = self.storage.retrieve_response( + spider, request + ) if cachedresponse is None: self.stats.inc_value("httpcache/miss", spider=spider) if self.ignore_missing: @@ -95,7 +102,7 @@ class HttpCacheMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Response: + ) -> Union[Request, Response]: if request.meta.get("dont_cache", False): return response @@ -110,7 +117,7 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse = request.meta.pop("cached_response", None) + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) @@ -126,8 +133,8 @@ class HttpCacheMiddleware: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Optional[Response]: - cachedresponse = request.meta.pop("cached_response", None) + ) -> Union[Request, Response, None]: + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 30a6952df..dfdad9864 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,14 +1,22 @@ -import io -import warnings -import zlib +from __future__ import annotations +import io +import zlib +from typing import TYPE_CHECKING, List, Optional, Union + +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.statscollectors import StatsCollector from scrapy.utils.gz import gunzip -ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: import brotli @@ -29,7 +37,7 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None, settings=None): + def __init__(self, stats: Optional[StatsCollector] = None): self.stats = stats if not stats: warnings.warn( @@ -61,7 +69,7 @@ class HttpCompressionMiddleware: ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured try: @@ -78,10 +86,15 @@ class HttpCompressionMiddleware: result.keep_encoding_header = False return result - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.method == "HEAD": return response if b'decoded' in response.flags: @@ -110,7 +123,7 @@ class HttpCompressionMiddleware: del response.headers['Content-Encoding'] return response - def _decode(self, body, encoding): + def _decode(self, body: bytes, encoding: bytes) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": body = gunzip(body) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index f74d84b69..335896ac1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,16 +1,30 @@ -import base64 -from urllib.parse import unquote, urlunparse -from urllib.request import _parse_proxy, getproxies, proxy_bypass +from __future__ import annotations +import base64 +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from urllib.parse import unquote, urlunparse +from urllib.request import ( # type: ignore[attr-defined] + _parse_proxy, + getproxies, + proxy_bypass, +) + +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpProxyMiddleware: - def __init__(self, auth_encoding="latin-1"): - self.auth_encoding = auth_encoding - self.proxies = {} + def __init__(self, auth_encoding: Optional[str] = "latin-1"): + self.auth_encoding: Optional[str] = auth_encoding + self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -20,19 +34,19 @@ class HttpProxyMiddleware: continue @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) - def _basic_auth_header(self, username, password): + def _basic_auth_header(self, username: str, password: str) -> bytes: user_pass = to_bytes( f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding ) return base64.b64encode(user_pass) - def _get_proxy(self, url, orig_type): + def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) @@ -43,7 +57,9 @@ class HttpProxyMiddleware: return creds, proxy_url - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: creds, proxy_url = None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: @@ -54,13 +70,16 @@ class HttpProxyMiddleware: if ( # 'no_proxy' is only supported by http schemes scheme not in ("http", "https") - or not proxy_bypass(parsed.hostname) + or (parsed.hostname and not proxy_bypass(parsed.hostname)) ) and scheme in self.proxies: creds, proxy_url = self.proxies[scheme] self._set_proxy_and_creds(request, proxy_url, creds) + return None - def _set_proxy_and_creds(self, request, proxy_url, creds): + def _set_proxy_and_creds( + self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + ) -> None: if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..814b1a561 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,17 +1,29 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, List, Union, cast from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _build_redirect_request(source_request, *, url, **kwargs): +def _build_redirect_request( + source_request: Request, *, url: str, **kwargs: Any +) -> Request: redirect_request = source_request.replace( url=url, **kwargs, @@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs): class BaseRedirectMiddleware: - enabled_setting = "REDIRECT_ENABLED" + enabled_setting: str = "REDIRECT_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool(self.enabled_setting): raise NotConfigured - self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES") - self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST") + self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") + self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def _redirect(self, redirected, request, spider, reason): + def _redirect( + self, redirected: Request, request: Request, spider: Spider, reason: Any + ) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 @@ -67,7 +81,9 @@ class BaseRedirectMiddleware: ) raise IgnoreRequest("max redirections reached") - def _redirect_request_using_get(self, request, redirect_url): + def _redirect_request_using_get( + self, request: Request, redirect_url: str + ) -> Request: redirect_request = _build_redirect_request( request, url=redirect_url, @@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware): and meta-refresh html tag. """ - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or response.status in getattr(spider, "handle_httpstatus_list", []) @@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if "Location" not in response.headers or response.status not in allowed_status: return response + assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): request_scheme = urlparse(request.url).scheme @@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = "METAREFRESH_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS") - self._maxdelay = settings.getint("METAREFRESH_MAXDELAY") + self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" @@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and interval < self._maxdelay: + if url and cast(float, interval) < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, "meta refresh") diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 081642a4b..3c494de78 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,41 +9,60 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +from __future__ import annotations + +import warnings from logging import Logger, getLogger -from typing import Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - ConnectionRefusedError, - DNSLookupError, - TCPTimedOutError, - TimeoutError, -) -from twisted.web.client import ResponseFailed - -from scrapy.core.downloader.handlers.http11 import TunnelError -from scrapy.exceptions import NotConfigured +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Response from scrapy.http.request import Request +from scrapy.settings import BaseSettings, Settings from scrapy.spiders import Spider +from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + retry_logger = getLogger(__name__) +def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: + if name == "EXCEPTIONS_TO_RETRY": + warnings.warn( + "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " + "Use the RETRY_EXCEPTIONS setting instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return tuple( + load_object(x) if isinstance(x, str) else x + for x in Settings().getlist("RETRY_EXCEPTIONS") + ) + raise AttributeError( + f"{self.__class__.__name__!r} object has no attribute {name!r}" + ) + + +class BackwardsCompatibilityMetaclass(type): + __getattr__ = backwards_compatibility_getattr + + def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception] = "unspecified", + reason: Union[str, Exception, Type[Exception]] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = "retry", -): +) -> Optional[Request]: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been @@ -85,6 +104,7 @@ def get_retry_request( retry-related job stats """ settings = spider.crawler.settings + assert spider.crawler.stats stats = spider.crawler.stats retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: @@ -121,24 +141,8 @@ def get_retry_request( return None -class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to - # decompress an empty response - EXCEPTIONS_TO_RETRY = ( - defer.TimeoutError, - TimeoutError, - DNSLookupError, - ConnectionRefusedError, - ConnectionDone, - ConnectError, - ConnectionLost, - TCPTimedOutError, - ResponseFailed, - IOError, - TunnelError, - ) - - def __init__(self, settings): +class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): + def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") @@ -147,11 +151,22 @@ class RetryMiddleware: ) self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + try: + self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") + except AttributeError: + # If EXCEPTIONS_TO_RETRY is not "overridden" + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) + @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: @@ -159,13 +174,21 @@ class RetryMiddleware: return self._retry(request, reason, spider) or response return response - def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: + if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) + return None - def _retry(self, request, reason, spider): + def _retry( + self, + request: Request, + reason: Union[str, Exception, Type[Exception]], + spider: Spider, + ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( @@ -175,3 +198,5 @@ class RetryMiddleware: max_retry_times=max_retry_times, priority_adjust=priority_adjust, ) + + __getattr__ = backwards_compatibility_getattr diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6cab27c5a..6a0ecb7bf 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK +from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class RobotsTxtMiddleware: - DOWNLOAD_PRIORITY = 1000 + DOWNLOAD_PRIORITY: int = 1000 - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None) - self.crawler = crawler - self._parsers = {} - self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER")) + self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") + self._robotstxt_useragent: Optional[str] = crawler.settings.get( + "ROBOTSTXT_USER_AGENT", None + ) + self.crawler: Crawler = crawler + self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parserimpl: RobotParser = load_object( + crawler.settings.get("ROBOTSTXT_PARSER") + ) # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: if request.meta.get("dont_obey_robotstxt"): - return + return None if request.url.startswith("data:") or request.url.startswith("file:"): - return - d = maybeDeferred(self.robot_parser, request, spider) + return None + d: Deferred = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d - def process_request_2(self, rp, request, spider): + def process_request_2( + self, rp: Optional[RobotParser], request: Request, spider: Spider + ) -> None: if rp is None: return - useragent = self._robotstxt_useragent + useragent: Union[str, bytes, None] = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) + assert useragent is not None if not rp.allowed(request.url, useragent): logger.debug( "Forbidden by robots.txt: %(request)s", {"request": request}, extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") - def robot_parser(self, request, spider): + def robot_parser( + self, request: Request, spider: Spider + ) -> Union[RobotParser, Deferred, None]: url = urlparse_cached(request) netloc = url.netloc @@ -75,24 +97,27 @@ class RobotsTxtMiddleware: meta={"dont_obey_robotstxt": True}, callback=NO_CALLBACK, ) + assert self.crawler.engine + assert self.crawler.stats dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) self.crawler.stats.inc_value("robotstxt/request_count") - if isinstance(self._parsers[netloc], Deferred): - d = Deferred() + parser = self._parsers[netloc] + if isinstance(parser, Deferred): + d: Deferred = Deferred() - def cb(result): + def cb(result: Any) -> Any: d.callback(result) return result - self._parsers[netloc].addCallback(cb) + parser.addCallback(cb) return d - return self._parsers[netloc] + return parser - def _logerror(self, failure, request, spider): + def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure: if failure.type is not IgnoreRequest: logger.error( "Error downloading %(request)s: %(f_exception)s", @@ -102,20 +127,24 @@ class RobotsTxtMiddleware: ) return failure - def _parse_robots(self, response, netloc, spider): + def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None: + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp rp_dfd.callback(rp) - def _robots_error(self, failure, netloc): + def _robots_error(self, failure: Failure, netloc: str) -> None: if failure.type is not IgnoreRequest: key = f"robotstxt/exception_count/{failure.type}" + assert self.crawler.stats self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None rp_dfd.callback(None) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index a0f62e262..df30e8ca4 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,11 +1,23 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Union + from twisted.web import http +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def get_header_size(headers): + +def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): @@ -14,30 +26,36 @@ def get_header_size(headers): return size + len(b"\r\n") * (len(headers.keys()) - 1) -def get_status_size(response_status): +def get_status_size(response_status: int) -> int: return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15 # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured + assert crawler.stats return cls(crawler.stats) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: self.stats.inc_value("downloader/request_count", spider=spider) self.stats.inc_value( f"downloader/request_method_count/{request.method}", spider=spider ) reqlen = len(request_httprepr(request)) self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: self.stats.inc_value("downloader/response_count", spider=spider) self.stats.inc_value( f"downloader/response_status_count/{response.status}", spider=spider @@ -52,9 +70,12 @@ class DownloaderStats: self.stats.inc_value("downloader/response_bytes", reslen, spider=spider) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count", spider=spider) self.stats.inc_value( f"downloader/exception_type_count/{ex_class}", spider=spider ) + return None diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 856a275ab..92f1ec897 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -1,23 +1,36 @@ """Set User-Agent header per spider or use a default value from settings""" -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" - def __init__(self, user_agent="Scrapy"): + def __init__(self, user_agent: str = "Scrapy"): self.user_agent = user_agent @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings["USER_AGENT"]) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.user_agent = getattr(spider, "user_agent", self.user_agent) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) + return None diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d796e5cbb..dd2420e98 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,25 +1,31 @@ +from __future__ import annotations + import logging from pathlib import Path -from typing import Optional, Set, Type, TypeVar -from warnings import warn +from typing import TYPE_CHECKING, Optional, Set from twisted.internet.defer import Deferred from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import RequestFingerprinter, referer_str +from scrapy.utils.request import ( + RequestFingerprinter, + RequestFingerprinterProtocol, + referer_str, +) -BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler class BaseDupeFilter: @classmethod - def from_settings( - cls: Type[BaseDupeFilterTV], settings: BaseSettings - ) -> BaseDupeFilterTV: + def from_settings(cls, settings: BaseSettings) -> Self: return cls() def request_seen(self, request: Request) -> bool: @@ -36,9 +42,6 @@ class BaseDupeFilter: pass -RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") - - class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" @@ -47,10 +50,12 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter=None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> None: self.file = None - self.fingerprinter = fingerprinter or RequestFingerprinter() + self.fingerprinter: RequestFingerprinterProtocol = ( + fingerprinter or RequestFingerprinter() + ) self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug @@ -62,40 +67,21 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_settings( - cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None - ) -> RFPDupeFilterTV: + cls, + settings: BaseSettings, + *, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, + ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") - try: - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their '__init__' " - "method to support a 'fingerprinter' parameter or reimplement " - "the 'from_settings' class method.", - ScrapyDeprecationWarning, - ) - result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter - return result + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @classmethod - def from_crawler(cls, crawler): - try: - return cls.from_settings( - crawler.settings, - fingerprinter=crawler.request_fingerprinter, - ) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their overridden " - "'__init__' method and 'from_settings' class method to " - "support a 'fingerprinter' parameter, or reimplement the " - "'from_crawler' class method.", - ScrapyDeprecationWarning, - ) - result = cls.from_settings(crawler.settings) - result.fingerprinter = crawler.request_fingerprinter - return result + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.request_fingerprinter + return cls.from_settings( + crawler.settings, + fingerprinter=crawler.request_fingerprinter, + ) def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) @@ -127,4 +113,5 @@ class RFPDupeFilter(BaseDupeFilter): self.logger.debug(msg, {"request": request}, extra={"spider": spider}) self.logdupes = False + assert spider.crawler.stats spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fedd02805..6d188c489 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from typing import Any # Internal @@ -51,7 +52,7 @@ class StopDownload(Exception): should be handled by the request errback. Note that 'fail' is a keyword-only argument. """ - def __init__(self, *, fail=True): + def __init__(self, *, fail: bool = True): super().__init__() self.fail = fail @@ -77,7 +78,7 @@ class NotSupported(Exception): class UsageError(Exception): """To indicate a command-line usage error""" - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): self.print_help = kw.pop("print_help", True) super().__init__(*a, **kw) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index bb3e3c662..f85f1dad8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,13 +7,11 @@ import io import marshal import pickle import pprint -import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import ItemAdapter, is_item -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -133,6 +131,13 @@ class JsonItemExporter(BaseItemExporter): if self.indent is not None: self.file.write(b"\n") + def _add_comma_after_first(self): + if self.first_item: + self.first_item = False + else: + self.file.write(b",") + self._beautify_newline() + def start_exporting(self): self.file.write(b"[") self._beautify_newline() @@ -142,14 +147,10 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item): - if self.first_item: - self.first_item = False - else: - self.file.write(b",") - self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) - data = self.encoder.encode(itemdict) - self.file.write(to_bytes(data, self.encoding)) + data = to_bytes(self.encoder.encode(itemdict), self.encoding) + self._add_comma_after_first() + self.file.write(data) class XmlItemExporter(BaseItemExporter): @@ -255,6 +256,9 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def finish_exporting(self): + self.stream.detach() # Avoid closing the wrapped file. + def _build_row(self, values): for s in values: try: @@ -324,13 +328,7 @@ class PythonItemExporter(BaseItemExporter): """ def _configure(self, options, dont_fail=False): - self.binary = options.pop("binary", True) super()._configure(options, dont_fail) - if self.binary: - warnings.warn( - "PythonItemExporter will drop support for binary export in the future", - ScrapyDeprecationWarning, - ) if not self.encoding: self.encoding = "utf-8" @@ -345,18 +343,14 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): - return encode_func(value, encoding=self.encoding) + return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item): for key, value in ItemAdapter(item).items(): - key = to_bytes(key) if self.binary else key yield key, self._serialize_value(value) def export_item(self, item): result = dict(self._get_serialized_fields(item)) - if self.binary: - result = dict(self._serialize_item(result)) return result diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index bb6f832f2..4307b4170 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,11 +4,14 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +import logging from collections import defaultdict from scrapy import signals from scrapy.exceptions import NotConfigured +logger = logging.getLogger(__name__) + class CloseSpider: def __init__(self, crawler): @@ -19,6 +22,7 @@ class CloseSpider: "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), + "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), } if not any(self.close_on.values()): @@ -34,6 +38,15 @@ class CloseSpider: crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + if self.close_on.get("timeout_no_item"): + self.timeout_no_item = self.close_on["timeout_no_item"] + self.items_in_period = 0 + crawler.signals.connect( + self.spider_opened_no_item, signal=signals.spider_opened + ) + crawler.signals.connect( + self.item_scraped_no_item, signal=signals.item_scraped + ) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -69,3 +82,31 @@ class CloseSpider: task = getattr(self, "task", False) if task and task.active(): task.cancel() + + task_no_item = getattr(self, "task_no_item", False) + if task_no_item and task_no_item.running: + task_no_item.stop() + + def spider_opened_no_item(self, spider): + from twisted.internet import task + + self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item.start(self.timeout_no_item, now=False) + + logger.info( + f"Spider will stop when no items are produced after " + f"{self.timeout_no_item} seconds." + ) + + def item_scraped_no_item(self, item, spider): + self.items_in_period += 1 + + def _count_items_produced(self, spider): + if self.items_in_period >= 1: + self.items_in_period = 0 + else: + logger.info( + f"Closing spider since no items were produced in the last " + f"{self.timeout_no_item} seconds." + ) + self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 30c987253..302a615f2 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,7 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime +from datetime import datetime, timezone from scrapy import signals @@ -22,11 +22,11 @@ class CoreStats: return o def spider_opened(self, spider): - self.start_time = datetime.utcnow() + self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) def spider_closed(self, spider, reason): - finish_time = datetime.utcnow() + finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() self.stats.set_value( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bcf0b779a..fadbbb582 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -8,10 +8,10 @@ import logging import re import sys import warnings -from datetime import datetime -from pathlib import Path +from datetime import datetime, timezone +from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -29,23 +29,20 @@ from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import get_func_args, without_none_values +from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) +try: + import boto3 # noqa: F401 + + IS_BOTO3_AVAILABLE = True +except ImportError: + IS_BOTO3_AVAILABLE = False + def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): - argument_names = get_func_args(builder) - if "feed_options" in argument_names: - kwargs["feed_options"] = feed_options - else: - warnings.warn( - f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove this " - "warning. This parameter will become mandatory in a future " - "version of Scrapy.", - category=ScrapyDeprecationWarning, - ) + kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -165,6 +162,7 @@ class S3FeedStorage(BlockingFeedStorage): *, feed_options=None, session_token=None, + region_name=None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -176,16 +174,41 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - import botocore.session + self.region_name = region_name + + if IS_BOTO3_AVAILABLE: + import boto3.session + + session = boto3.session.Session() + + self.s3_client = session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) + else: + warnings.warn( + "`botocore` usage has been deprecated for S3 feed " + "export, please use `boto3` to avoid problems", + category=ScrapyDeprecationWarning, + ) + + import botocore.session + + session = botocore.session.get_session() + + self.s3_client = session.create_client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) - session = botocore.session.get_session() - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - ) if feed_options and feed_options.get("overwrite", True) is False: logger.warning( "S3 does not support appending to files. To " @@ -203,15 +226,22 @@ class S3FeedStorage(BlockingFeedStorage): session_token=crawler.settings["AWS_SESSION_TOKEN"], acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, + region_name=crawler.settings["AWS_REGION_NAME"] or None, feed_options=feed_options, ) def _store_in_thread(self, file): file.seek(0) - kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + if IS_BOTO3_AVAILABLE: + kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) + else: + kwargs = {"ACL": self.acl} if self.acl else {} + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + ) file.close() @@ -242,15 +272,23 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False, *, feed_options=None): + def __init__( + self, + uri: str, + use_active_mode: bool = False, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): u = urlparse(uri) - self.host = u.hostname - self.port = int(u.port or "21") - self.username = u.username - self.password = unquote(u.password or "") - self.path = u.path - self.use_active_mode = use_active_mode - self.overwrite = not feed_options or feed_options.get("overwrite", True) + if not u.hostname: + raise ValueError(f"Got a storage URI without a hostname: {uri}") + self.host: str = u.hostname + self.port: int = int(u.port or "21") + self.username: str = u.username or "" + self.password: str = unquote(u.password or "") + self.path: str = u.path + self.use_active_mode: bool = use_active_mode + self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): @@ -277,8 +315,6 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - file, - exporter, storage, uri, format, @@ -286,9 +322,14 @@ class FeedSlot: batch_id, uri_template, filter, + feed_options, + spider, + exporters, + settings, + crawler, ): - self.file = file - self.exporter = exporter + self.file = None + self.exporter = None self.storage = storage # feed params self.batch_id = batch_id @@ -297,15 +338,44 @@ class FeedSlot: self.uri_template = uri_template self.uri = uri self.filter = filter + # exporter params + self.feed_options = feed_options + self.spider = spider + self.exporters = exporters + self.settings = settings + self.crawler = crawler # flags self.itemcount = 0 self._exporting = False + self._fileloaded = False def start_exporting(self): + if not self._fileloaded: + self.file = self.storage.open(self.spider) + if "postprocessing" in self.feed_options: + self.file = PostProcessingManager( + self.feed_options["postprocessing"], self.file, self.feed_options + ) + self.exporter = self._get_exporter( + file=self.file, + format=self.feed_options["format"], + fields_to_export=self.feed_options["fields"], + encoding=self.feed_options["encoding"], + indent=self.feed_options["indent"], + **self.feed_options["item_export_kwargs"], + ) + self._fileloaded = True + if not self._exporting: self.exporter.start_exporting() self._exporting = True + def _get_instance(self, objcls, *args, **kwargs): + return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + + def _get_exporter(self, file, format, *args, **kwargs): + return self._get_instance(self.exporters[format], file, *args, **kwargs) + def finish_exporting(self): if self._exporting: self.exporter.finish_exporting() @@ -347,7 +417,9 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) - uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects + uri = self.settings["FEED_URI"] + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings @@ -357,7 +429,8 @@ class FeedExporter: # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict("FEEDS").items(): - uri = str(uri) # handle pathlib.Path objects + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) @@ -406,11 +479,16 @@ class FeedExporter: return slot_.file.file return slot_.file - slot.finish_exporting() - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, get_file(slot)) + if slot.itemcount: + # Normal case + slot.finish_exporting() + elif slot.store_empty and slot.batch_id == 1: + # Need to store the empty file + slot.start_exporting() + slot.finish_exporting() + else: + # In this case, the file is not stored, so no processing is required. + return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, get_file(slot)) @@ -455,23 +533,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - file = storage.open(spider) - if "postprocessing" in feed_options: - file = PostProcessingManager( - feed_options["postprocessing"], file, feed_options - ) - - exporter = self._get_exporter( - file=file, - format=feed_options["format"], - fields_to_export=feed_options["fields"], - encoding=feed_options["encoding"], - indent=feed_options["indent"], - **feed_options["item_export_kwargs"], - ) slot = FeedSlot( - file=file, - exporter=exporter, storage=storage, uri=uri, format=feed_options["format"], @@ -479,9 +541,12 @@ class FeedExporter: batch_id=batch_id, uri_template=uri_template, filter=self.filters[uri_template], + feed_options=feed_options, + spider=spider, + exporters=self.exporters, + settings=self.settings, + crawler=getattr(self, "crawler", None), ) - if slot.store_empty: - slot.start_exporting() return slot def item_scraped(self, item, spider): @@ -553,7 +618,7 @@ class FeedExporter: def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme - if scheme in self.storages: + if scheme in self.storages or PureWindowsPath(uri).drive: try: self._get_storage(uri, feed_options) return True @@ -565,21 +630,13 @@ class FeedExporter: else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) - def _get_instance(self, objcls, *args, **kwargs): - return create_instance( - objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs - ) - - def _get_exporter(self, file, format, *args, **kwargs): - return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri, feed_options): """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that do not support it, and issuing a deprecation warning instead. """ - feedcls = self.storages[urlparse(uri).scheme] + feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): @@ -609,25 +666,16 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - utc_now = datetime.utcnow() + utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - original_params = params.copy() uripar_function = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params ) new_params = uripar_function(params, spider) - if new_params is None or original_params != params: - warnings.warn( - "Modifying the params dictionary in-place in the function defined in " - "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS " - "setting is deprecated. The function must return a new dictionary " - "instead.", - category=ScrapyDeprecationWarning, - ) return new_params if new_params is not None else params def _load_filter(self, feed_options): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dfe843974..7e4f047a8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -291,6 +291,7 @@ class FilesystemCacheStorage: extra={"spider": spider}, ) + assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter def close_spider(self, spider): diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py new file mode 100644 index 000000000..2d557f123 --- /dev/null +++ b/scrapy/extensions/periodic_log.py @@ -0,0 +1,140 @@ +import logging +from datetime import datetime, timezone + +from twisted.internet import task + +from scrapy import signals +from scrapy.exceptions import NotConfigured +from scrapy.utils.serialize import ScrapyJSONEncoder + +logger = logging.getLogger(__name__) + + +class PeriodicLog: + """Log basic scraping stats periodically""" + + def __init__( + self, + stats, + interval=60.0, + ext_stats={}, + ext_delta={}, + ext_timing_enabled=False, + ): + self.stats = stats + self.interval = interval + self.multiplier = 60.0 / self.interval + self.task = None + self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled = bool(ext_stats) + self.ext_stats_include = ext_stats.get("include", []) + self.ext_stats_exclude = ext_stats.get("exclude", []) + self.ext_delta_enabled = bool(ext_delta) + self.ext_delta_include = ext_delta.get("include", []) + self.ext_delta_exclude = ext_delta.get("exclude", []) + self.ext_timing_enabled = ext_timing_enabled + + @classmethod + def from_crawler(cls, crawler): + interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + if not interval: + raise NotConfigured + try: + ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + except (TypeError, ValueError): + ext_stats = ( + {"enabled": True} + if crawler.settings.getbool("PERIODIC_LOG_STATS") + else None + ) + try: + ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + except (TypeError, ValueError): + ext_delta = ( + {"enabled": True} + if crawler.settings.getbool("PERIODIC_LOG_DELTA") + else None + ) + + ext_timing_enabled = crawler.settings.getbool( + "PERIODIC_LOG_TIMING_ENABLED", False + ) + if not (ext_stats or ext_delta or ext_timing_enabled): + raise NotConfigured + o = cls( + crawler.stats, + interval, + ext_stats, + ext_delta, + ext_timing_enabled, + ) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o + + def spider_opened(self, spider): + self.time_prev = datetime.now(tz=timezone.utc) + self.delta_prev = {} + self.stats_prev = {} + + self.task = task.LoopingCall(self.log) + self.task.start(self.interval) + + def log(self): + data = {} + if self.ext_timing_enabled: + data.update(self.log_timing()) + if self.ext_delta_enabled: + data.update(self.log_delta()) + if self.ext_stats_enabled: + data.update(self.log_crawler_stats()) + logger.info(self.encoder.encode(data)) + + def log_delta(self): + num_stats = { + k: v + for k, v in self.stats._stats.items() + if isinstance(v, (int, float)) + and self.param_allowed(k, self.ext_delta_include, self.ext_delta_exclude) + } + delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()} + self.delta_prev = num_stats + return {"delta": delta} + + def log_timing(self): + now = datetime.now(tz=timezone.utc) + time = { + "log_interval": self.interval, + "start_time": self.stats._stats["start_time"], + "utcnow": now, + "log_interval_real": (now - self.time_prev).total_seconds(), + "elapsed": (now - self.stats._stats["start_time"]).total_seconds(), + } + self.time_prev = now + return {"time": time} + + def log_crawler_stats(self): + stats = { + k: v + for k, v in self.stats._stats.items() + if self.param_allowed(k, self.ext_stats_include, self.ext_stats_exclude) + } + return {"stats": stats} + + def param_allowed(self, stat_name, include, exclude): + if not include and not exclude: + return True + for p in exclude: + if p in stat_name: + return False + if exclude and not include: + return True + for p in include: + if p in stat_name: + return True + return False + + def spider_closed(self, spider, reason): + self.log() + if self.task and self.task.running: + self.task.stop() diff --git a/scrapy/http/common.py b/scrapy/http/common.py deleted file mode 100644 index bc8861574..000000000 --- a/scrapy/http/common.py +++ /dev/null @@ -1,7 +0,0 @@ -def obsolete_setter(setter, attrname): - def newsetter(self, value): - c = self.__class__.__name__ - msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" - raise AttributeError(msg) - - return newsetter diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a5329ad51..2595f328e 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,8 +1,12 @@ import re import time +from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import DefaultCookiePolicy +from typing import Sequence +from scrapy import Request +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -24,7 +28,7 @@ class CookieJar: wrsp = WrappedResponse(response) return self.jar.extract_cookies(wrsp, wreq) - def add_cookie_header(self, request): + def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) self.policy._now = self.jar._now = int(time.time()) @@ -75,7 +79,7 @@ class CookieJar: def set_policy(self, pol): return self.jar.set_policy(pol) - def make_cookies(self, response, request): + def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) return self.jar.make_cookies(wrsp, wreq) @@ -83,7 +87,7 @@ class CookieJar: def set_cookie(self, cookie): self.jar.set_cookie(cookie) - def set_cookie_if_ok(self, cookie, request): + def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 2540be01a..21eb9fb73 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,41 +1,73 @@ +from __future__ import annotations + from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from w3lib.http import headers_dict_to_raw -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +_RawValueT = Union[bytes, str, int] + + +# isn't fully compatible typing-wise with either dict or CaselessDict, +# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146 class Headers(CaselessDict): """Case insensitive http headers dictionary""" - def __init__(self, seq=None, encoding="utf-8"): - self.encoding = encoding + def __init__( + self, + seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + encoding: str = "utf-8", + ): + self.encoding: str = encoding super().__init__(seq) - def update(self, seq): + def update( # type: ignore[override] + self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq = {} + iseq: Dict[bytes, List[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) - def normkey(self, key): + def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value): + def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: """Normalize values to bytes""" + _value: Iterable[_RawValueT] if value is None: - value = [] + _value = [] elif isinstance(value, (str, bytes)): - value = [value] - elif not hasattr(value, "__iter__"): - value = [value] + _value = [value] + elif hasattr(value, "__iter__"): + _value = value + else: + _value = [value] - return [self._tobytes(x) for x in value] + return [self._tobytes(x) for x in _value] - def _tobytes(self, x): + def _tobytes(self, x: _RawValueT) -> bytes: if isinstance(x, bytes): return x if isinstance(x, str): @@ -44,51 +76,54 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key): + def __getitem__(self, key: AnyStr) -> Optional[bytes]: try: - return super().__getitem__(key)[-1] + return cast(List[bytes], super().__getitem__(key))[-1] except IndexError: return None - def get(self, key, def_val=None): + def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: try: - return super().get(key, def_val)[-1] + return cast(List[bytes], super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key, def_val=None): + def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: try: - return super().__getitem__(key) + return cast(List[bytes], super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) return [] - def setlist(self, key, list_): + def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None: self[key] = list_ - def setlistdefault(self, key, default_list=()): + def setlistdefault( + self, key: AnyStr, default_list: Iterable[_RawValueT] = () + ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key, value): + def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst - def items(self): + def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self): + def values(self) -> List[Optional[bytes]]: # type: ignore[override] return [self[k] for k in self.keys()] - def to_string(self): - return headers_dict_to_raw(self) + def to_string(self) -> bytes: + # cast() can be removed if the headers_dict_to_raw() hint is improved + return cast(bytes, headers_dict_to_raw(self)) - def to_unicode_dict(self): - """Return headers as a CaselessDict with unicode keys - and unicode values. Multiple values are joined with ','. + def to_unicode_dict(self) -> CaseInsensitiveDict: + """Return headers as a CaseInsensitiveDict with str keys + and str values. Multiple values are joined with ','. """ - return CaselessDict( + return CaseInsensitiveDict( ( to_unicode(key, encoding=self.encoding), to_unicode(b",".join(value), encoding=self.encoding), @@ -96,7 +131,7 @@ class Headers(CaselessDict): for key, value in self.items() ) - def __copy__(self): + def __copy__(self) -> Self: return self.__class__(self) copy = __copy__ diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9ba6ddf20..a1c5a5e51 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,12 +5,26 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing import ( + Any, + AnyStr, + Callable, + Dict, + Iterable, + List, + Mapping, + NoReturn, + Optional, + Tuple, + Type, + TypeVar, + Union, + cast, +) from w3lib.url import safe_url_string import scrapy -from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes @@ -20,7 +34,7 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -def NO_CALLBACK(*args, **kwargs): +def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of :class:`~scrapy.http.Request`, it indicates that the request is not meant to have a spider callback at all. @@ -77,24 +91,24 @@ class Request(object_ref): url: str, callback: Optional[Callable] = None, method: str = "GET", - headers: Optional[dict] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[dict] = None, + meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable] = None, flags: Optional[List[str]] = None, - cb_kwargs: Optional[dict] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - self._encoding = encoding # this one has to be set first - self.method = str(method).upper() + self._encoding: str = encoding # this one has to be set first + self.method: str = str(method).upper() self._set_url(url) self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") - self.priority = priority + self.priority: int = priority if not (callable(callback) or callback is None): raise TypeError( @@ -102,30 +116,33 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback = callback - self.errback = errback + self.callback: Optional[Callable] = callback + self.errback: Optional[Callable] = errback - self.cookies = cookies or {} - self.headers = Headers(headers or {}, encoding=encoding) - self.dont_filter = dont_filter + self.cookies: Union[dict, List[dict]] = cookies or {} + self.headers: Headers = Headers(headers or {}, encoding=encoding) + self.dont_filter: bool = dont_filter - self._meta = dict(meta) if meta else None - self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None - self.flags = [] if flags is None else list(flags) + self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[Dict[str, Any]] = ( + dict(cb_kwargs) if cb_kwargs else None + ) + self.flags: List[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> dict: + def cb_kwargs(self) -> Dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> dict: + def meta(self) -> Dict[str, Any]: if self._meta is None: self._meta = {} return self._meta - def _get_url(self) -> str: + @property + def url(self) -> str: return self._url def _set_url(self, url: str) -> None: @@ -142,16 +159,13 @@ class Request(object_ref): ): raise ValueError(f"Missing scheme in request url: {self._url}") - url = property(_get_url, obsolete_setter(_set_url, "url")) - - def _get_body(self) -> bytes: + @property + def body(self) -> bytes: return self._body def _set_body(self, body: Optional[Union[str, bytes]]) -> None: self._body = b"" if body is None else to_bytes(body, self.encoding) - body = property(_get_body, obsolete_setter(_set_body, "body")) - @property def encoding(self) -> str: return self._encoding @@ -162,19 +176,19 @@ class Request(object_ref): def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop("cls", self.__class__) - return cls(*args, **kwargs) + return cast(Request, cls(*args, **kwargs)) @classmethod def from_curl( cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, - **kwargs, + **kwargs: Any, ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the @@ -188,7 +202,7 @@ class Request(object_ref): ``ignore_unknown_options=False``. .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` - subclasses, such as :class:`~scrapy.http.JSONRequest`, or + subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares ` and @@ -207,7 +221,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. @@ -232,7 +246,7 @@ class Request(object_ref): return d -def _find_method(obj, func): +def _find_method(obj: Any, func: Callable) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8da779b7c..0f80a0ab7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,9 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import ( @@ -24,7 +26,10 @@ from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes from scrapy.utils.response import get_base_url -FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + FormdataKVType = Tuple[str, Union[str, Iterable[str]]] FormdataType = Optional[Union[dict, List[FormdataKVType]]] @@ -33,7 +38,9 @@ FormdataType = Optional[Union[dict, List[FormdataKVType]]] class FormRequest(Request): valid_form_methods = ["GET", "POST"] - def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: + def __init__( + self, *args: Any, formdata: FormdataType = None, **kwargs: Any + ) -> None: if formdata and kwargs.get("method") is None: kwargs["method"] = "POST" @@ -54,7 +61,7 @@ class FormRequest(Request): @classmethod def from_response( - cls: Type[FormRequestTypeVar], + cls, response: TextResponse, formname: Optional[str] = None, formid: Optional[str] = None, @@ -64,8 +71,8 @@ class FormRequest(Request): dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, - **kwargs, - ) -> FormRequestTypeVar: + **kwargs: Any, + ) -> Self: kwargs.setdefault("encoding", response.encoding) if formcss is not None: @@ -121,12 +128,12 @@ def _get_form( if formname is not None: f = root.xpath(f'//form[@name="{formname}"]') if f: - return f[0] + return cast(FormElement, f[0]) if formid is not None: f = root.xpath(f'//form[@id="{formid}"]') if f: - return f[0] + return cast(FormElement, f[0]) # Get form element from xpath, if not found, go up if formxpath is not None: @@ -135,7 +142,7 @@ def _get_form( el = nodes[0] while True: if el.tag == "form": - return el + return cast(FormElement, el) el = el.getparent() if el is None: break @@ -147,7 +154,7 @@ def _get_form( except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") else: - return form + return cast(FormElement, form) def _get_inputs( diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 510c903db..1dd9e6c87 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,16 +8,17 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Optional, Tuple +from typing import Any, Optional, Tuple from scrapy.http.request import Request -from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + def __init__( + self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs = dumps_kwargs @@ -43,7 +44,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> Request: + def replace(self, *args: Any, **kwargs: Any) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None @@ -58,6 +59,3 @@ class JsonRequest(Request): def _dumps(self, data: dict) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) - - -JSONRequest = create_deprecated_class("JSONRequest", JsonRequest) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 43692923b..bde860a66 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,7 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib -from typing import Optional +from typing import Any, Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -14,7 +14,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, encoding: Optional[str] = None, **kwargs): + def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs["body"] = xmlrpclib.dumps(**kw) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a82ed834a..6eae3e8b3 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,16 +4,37 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Generator, Tuple +from __future__ import annotations + +from ipaddress import IPv4Address, IPv6Address +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Callable, + Dict, + Generator, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urljoin +from twisted.internet.ssl import Certificate + from scrapy.exceptions import NotSupported -from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.link import Link from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + from scrapy.selector import SelectorList + class Response(object_ref): """An object that represents an HTTP response, which is usually @@ -41,29 +62,29 @@ class Response(object_ref): def __init__( self, url: str, - status=200, - headers=None, - body=b"", - flags=None, - request=None, - certificate=None, - ip_address=None, - protocol=None, + status: int = 200, + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: bytes = b"", + flags: Optional[List[str]] = None, + request: Optional[Request] = None, + certificate: Optional[Certificate] = None, + ip_address: Union[IPv4Address, IPv6Address, None] = None, + protocol: Optional[str] = None, ): - self.headers = Headers(headers or {}) - self.status = int(status) + self.headers: Headers = Headers(headers or {}) + self.status: int = int(status) self._set_body(body) self._set_url(url) - self.request = request - self.flags = [] if flags is None else list(flags) - self.certificate = certificate - self.ip_address = ip_address - self.protocol = protocol + self.request: Optional[Request] = request + self.flags: List[str] = [] if flags is None else list(flags) + self.certificate: Optional[Certificate] = certificate + self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address + self.protocol: Optional[str] = protocol @property - def cb_kwargs(self): + def cb_kwargs(self) -> Dict[str, Any]: try: - return self.request.cb_kwargs + return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: raise AttributeError( "Response.cb_kwargs not available, this response " @@ -71,32 +92,32 @@ class Response(object_ref): ) @property - def meta(self): + def meta(self) -> Dict[str, Any]: try: - return self.request.meta + return self.request.meta # type: ignore[union-attr] except AttributeError: raise AttributeError( "Response.meta not available, this response " "is not tied to any request" ) - def _get_url(self): + @property + def url(self) -> str: return self._url - def _set_url(self, url: str): + def _set_url(self, url: str) -> None: if isinstance(url, str): - self._url = url + self._url: str = url else: raise TypeError( f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" ) - url = property(_get_url, obsolete_setter(_set_url, "url")) - - def _get_body(self): + @property + def body(self) -> bytes: return self._body - def _set_body(self, body): + def _set_body(self, body: Optional[bytes]) -> None: if body is None: self._body = b"" elif not isinstance(body, bytes): @@ -108,47 +129,45 @@ class Response(object_ref): else: self._body = body - body = property(_get_body, obsolete_setter(_set_body, "body")) - - def __repr__(self): + def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self): + def copy(self) -> Response: """Return a copy of this Response""" return self.replace() - def replace(self, *args, **kwargs): + def replace(self, *args: Any, **kwargs: Any) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop("cls", self.__class__) - return cls(*args, **kwargs) + return cast(Response, cls(*args, **kwargs)) - def urljoin(self, url): + def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" return urljoin(self.url, url) @property - def text(self): + def text(self) -> str: """For subclasses of TextResponse, this will return the body as str """ raise AttributeError("Response content isn't text") - def css(self, *a, **kw): + def css(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") - def jmespath(self, *a, **kw): + def jmespath(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") - def xpath(self, *a, **kw): + def xpath(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ @@ -156,19 +175,19 @@ class Response(object_ref): def follow( self, - url, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding="utf-8", - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + url: Union[str, Link], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -183,6 +202,8 @@ class Response(object_ref): .. versionadded:: 2.0 The *flags* parameter. """ + if encoding is None: + raise ValueError("encoding can't be None") if isinstance(url, Link): url = url.url elif url is None: @@ -207,19 +228,19 @@ class Response(object_ref): def follow_all( self, - urls, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding="utf-8", - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + urls: Iterable[Union[str, Link]], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5289f014a..6596d8a5c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,10 +4,25 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations import json from contextlib import suppress -from typing import Generator, Tuple +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Callable, + Dict, + Generator, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urljoin import parsel @@ -22,9 +37,13 @@ from w3lib.html import strip_html5_whitespace from scrapy.http import Request from scrapy.http.response import Response +from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +if TYPE_CHECKING: + from scrapy.selector import Selector, SelectorList + _NONE = object() @@ -34,21 +53,15 @@ class TextResponse(Response): attributes: Tuple[str, ...] = Response.attributes + ("encoding",) - def __init__(self, *args, **kwargs): - self._encoding = kwargs.pop("encoding", None) - self._cached_benc = None - self._cached_ubody = None - self._cached_selector = None + def __init__(self, *args: Any, **kwargs: Any): + self._encoding: Optional[str] = kwargs.pop("encoding", None) + self._cached_benc: Optional[str] = None + self._cached_ubody: Optional[str] = None + self._cached_selector: Optional[Selector] = None super().__init__(*args, **kwargs) - def _set_url(self, url): - if isinstance(url, str): - self._url = to_unicode(url, self.encoding) - else: - super()._set_url(url) - - def _set_body(self, body): - self._body = b"" # used by encoding detection + def _set_body(self, body: Union[str, bytes, None]) -> None: + self._body: bytes = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: raise TypeError( @@ -60,10 +73,10 @@ class TextResponse(Response): super()._set_body(body) @property - def encoding(self): + def encoding(self) -> str: return self._declared_encoding() or self._body_inferred_encoding() - def _declared_encoding(self): + def _declared_encoding(self) -> Optional[str]: return ( self._encoding or self._bom_encoding() @@ -71,18 +84,18 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def json(self): + def json(self) -> Any: """ .. versionadded:: 2.2 Deserialize a JSON document to a Python object. """ if self._cached_decoded_json is _NONE: - self._cached_decoded_json = json.loads(self.text) + self._cached_decoded_json = json.loads(self.body) return self._cached_decoded_json @property - def text(self): + def text(self) -> str: """Body as unicode""" # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called @@ -92,20 +105,20 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody - def urljoin(self, url): + def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" return urljoin(get_base_url(self), url) @memoizemethod_noargs - def _headers_encoding(self): - content_type = self.headers.get(b"Content-Type", b"") + def _headers_encoding(self) -> Optional[str]: + content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) - def _body_inferred_encoding(self): + def _body_inferred_encoding(self) -> str: if self._cached_benc is None: content_type = to_unicode( - self.headers.get(b"Content-Type", b""), encoding="latin-1" + cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1" ) benc, ubody = html_to_unicode( content_type, @@ -117,59 +130,66 @@ class TextResponse(Response): self._cached_ubody = ubody return self._cached_benc - def _auto_detect_fun(self, text): + def _auto_detect_fun(self, text: bytes) -> Optional[str]: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) except UnicodeError: continue return resolve_encoding(enc) + return None @memoizemethod_noargs - def _body_declared_encoding(self): + def _body_declared_encoding(self) -> Optional[str]: return html_body_declared_encoding(self.body) @memoizemethod_noargs - def _bom_encoding(self): + def _bom_encoding(self) -> Optional[str]: return read_bom(self.body)[0] @property - def selector(self): + def selector(self) -> Selector: from scrapy.selector import Selector if self._cached_selector is None: self._cached_selector = Selector(self) return self._cached_selector - def jmespath(self, query, **kwargs): + def jmespath(self, query: str, **kwargs: Any) -> SelectorList: + from scrapy.selector import SelectorList + if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined] + return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] - def xpath(self, query, **kwargs): - return self.selector.xpath(query, **kwargs) + def xpath(self, query: str, **kwargs: Any) -> SelectorList: + from scrapy.selector import SelectorList - def css(self, query): - return self.selector.css(query) + return cast(SelectorList, self.selector.xpath(query, **kwargs)) + + def css(self, query: str) -> SelectorList: + from scrapy.selector import SelectorList + + return cast(SelectorList, self.selector.css(query)) def follow( self, - url, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding=None, - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + url: Union[str, Link, parsel.Selector], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = None, + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -210,21 +230,21 @@ class TextResponse(Response): def follow_all( self, - urls=None, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding=None, - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, - css=None, - xpath=None, + urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = None, + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, + css: Optional[str] = None, + xpath: Optional[str] = None, ) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all @@ -266,7 +286,7 @@ class TextResponse(Response): with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) return super().follow_all( - urls=urls, + urls=cast(Iterable[Union[str, Link]], urls), callback=callback, method=method, headers=headers, @@ -288,8 +308,7 @@ class _InvalidSelector(ValueError): """ -def _url_from_selector(sel): - # type: (parsel.Selector) -> str +def _url_from_selector(sel: parsel.Selector) -> str: if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) diff --git a/scrapy/link.py b/scrapy/link.py index 704649731..0868ae5ef 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ +from typing import Any class Link: @@ -26,16 +27,20 @@ class Link: __slots__ = ["url", "text", "fragment", "nofollow"] - def __init__(self, url, text="", fragment="", nofollow=False): + def __init__( + self, url: str, text: str = "", fragment: str = "", nofollow: bool = False + ): if not isinstance(url, str): got = url.__class__.__name__ raise TypeError(f"Link urls must be str objects, got {got}") - self.url = url - self.text = text - self.fragment = fragment - self.nofollow = nofollow + self.url: str = url + self.text: str = text + self.fragment: str = fragment + self.nofollow: bool = nofollow - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Link): + raise NotImplementedError return ( self.url == other.url and self.text == other.text @@ -43,12 +48,12 @@ class Link: and self.nofollow == other.nofollow ) - def __hash__(self): + def __hash__(self) -> int: return ( hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) ) - def __repr__(self): + def __repr__(self) -> str: return ( f"Link(url={self.url!r}, text={self.text!r}, " f"fragment={self.fragment!r}, nofollow={self.nofollow!r})" diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ae2948d73..3774430a7 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -38,6 +38,7 @@ IGNORED_EXTENSIONS = [ "svg", "cdr", "ico", + "webp", # audio "mp3", "wma", @@ -66,12 +67,22 @@ IGNORED_EXTENSIONS = [ "webm", # office suites "xls", + "xlsm", "xlsx", + "xltm", + "xltx", + "potm", + "potx", "ppt", + "pptm", "pptx", "pps", "doc", + "docb", + "docm", "docx", + "dotm", + "dotx", "odt", "ods", "odg", @@ -85,6 +96,16 @@ IGNORED_EXTENSIONS = [ "dmg", "iso", "apk", + "jar", + "sh", + "rb", + "js", + "hta", + "bat", + "cpl", + "msi", + "msp", + "py", ] diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 91337b949..1042a3d48 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -78,7 +78,7 @@ class ItemLoader(itemloaders.ItemLoader): read-only. """ - default_item_class = Item + default_item_class: type = Item default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, parent=None, **context): diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py deleted file mode 100644 index 3e8644e0c..000000000 --- a/scrapy/loader/common.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Common functions used in Item Loaders code""" - -import warnings - -from itemloaders import common - -from scrapy.utils.deprecate import ScrapyDeprecationWarning - - -def wrap_loader_context(function, context): - """Wrap functions that receive loader_context to contain the context - "pre-loaded" and expose a interface that receives only one argument - """ - warnings.warn( - "scrapy.loader.common.wrap_loader_context has moved to a new library." - "Please update your reference to itemloaders.common.wrap_loader_context", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py deleted file mode 100644 index b82c6d5c7..000000000 --- a/scrapy/loader/processors.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This module provides some commonly used processors for Item Loaders. - -See documentation in docs/topics/loaders.rst -""" -from itemloaders import processors - -from scrapy.utils.deprecate import create_deprecated_class - -MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) - -Compose = create_deprecated_class("Compose", processors.Compose) - -TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst) - -Identity = create_deprecated_class("Identity", processors.Identity) - -SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes) - -Join = create_deprecated_class("Join", processors.Join) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 7cb379b46..d720b2f38 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging import os -from typing import Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.python.failure import Failure @@ -8,6 +10,13 @@ from scrapy import Request, Spider from scrapy.http import Response from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" @@ -105,7 +114,7 @@ class LogFormatter: } def item_error( - self, item: Any, exception, response: Response, spider: Spider + self, item: Any, exception: BaseException, response: Response, spider: Spider ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -121,7 +130,11 @@ class LogFormatter: } def spider_error( - self, failure: Failure, request: Request, response: Response, spider: Spider + self, + failure: Failure, + request: Request, + response: Union[Response, Failure], + spider: Spider, ) -> dict: """Logs an error message from a spider. @@ -161,5 +174,5 @@ class LogFormatter: } @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls() diff --git a/scrapy/mail.py b/scrapy/mail.py index c11f3898d..237327451 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -96,10 +96,9 @@ class MailSender: rcpts.extend(cc) msg["Cc"] = COMMASPACE.join(cc) - if charset: - msg.set_charset(charset) - if attachs: + if charset: + msg.set_charset(charset) msg.attach(MIMEText(body, "plain", charset or "us-ascii")) for attach_name, mimetype, f in attachs: part = MIMEBase(*mimetype.split("/")) @@ -110,7 +109,7 @@ class MailSender: ) msg.attach(part) else: - msg.set_payload(body) + msg.set_payload(body, charset) if _callback: _callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..090588130 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,21 @@ +from __future__ import annotations + import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from twisted.internet.defer import Deferred @@ -11,6 +25,13 @@ from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) @@ -34,7 +55,9 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings(cls, settings: Settings, crawler=None): + def from_settings( + cls, settings: Settings, crawler: Optional[Crawler] = None + ) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -46,10 +69,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) @@ -64,7 +86,7 @@ class MiddlewareManager: return cls(*middlewares) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls.from_settings(crawler.settings, crawler) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b594ccb7..5c09ab37e 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import md5sum @@ -155,7 +155,7 @@ class S3FilesStore: def _headers_to_botocore_kwargs(self, headers): """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. - mapping = CaselessDict( + mapping = CaseInsensitiveDict( { "Content-Type": "ContentType", "Cache-Control": "CacheControl", diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9d18144ee..1bd9832a8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,6 +8,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO +from typing import Dict, Tuple from itemadapter import ItemAdapter @@ -48,7 +49,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH = 0 MIN_HEIGHT = 0 EXPIRES = 90 - THUMBS = {} + THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 401029439..75532034a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,8 +1,6 @@ import functools import logging from collections import defaultdict -from inspect import signature -from warnings import warn from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -11,7 +9,6 @@ from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter @@ -44,9 +41,6 @@ class MediaPipeline: self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) self._handle_statuses(self.allow_redirects) - # Check if deprecated methods are being used and make them compatible - self._make_compatible() - def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -126,52 +120,6 @@ class MediaPipeline: ) return dfd.addBoth(lambda _: wad) # it must return wad at last - def _make_compatible(self): - """Make overridable methods of MediaPipeline and subclasses backwards compatible""" - methods = [ - "file_path", - "thumb_path", - "media_to_download", - "media_downloaded", - "file_downloaded", - "image_downloaded", - "get_images", - ] - - for method_name in methods: - method = getattr(self, method_name, None) - if callable(method): - setattr(self, method_name, self._compatible(method)) - - def _compatible(self, func): - """Wrapper for overridable methods to allow backwards compatibility""" - self._check_signature(func) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if self._expects_item[func.__name__]: - return func(*args, **kwargs) - - kwargs.pop("item", None) - return func(*args, **kwargs) - - return wrapper - - def _check_signature(self, func): - sig = signature(func) - self._expects_item[func.__name__] = True - - if "item" not in sig.parameters: - old_params = str(sig)[1:-1] - new_params = old_params + ", *, item=None" - warn( - f"{func.__name__}(self, {old_params}) is deprecated, " - f"please use {func.__name__}(self, {new_params})", - ScrapyDeprecationWarning, - stacklevel=2, - ) - self._expects_item[func.__name__] = False - def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list diff --git a/scrapy/py.typed b/scrapy/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6cbe01cbf..e2e8beff4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,3 +1,5 @@ +from typing import Any + from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( @@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache # TODO: cache misses -dnscache = LocalCache(10000) +dnscache: LocalCache[str, Any] = LocalCache(10000) @implementer(IResolverSimple) @@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver): def install_on_reactor(self): self.reactor.installResolver(self) - def getHostByName(self, name, timeout=None): + def getHostByName(self, name: str, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with @@ -110,7 +112,7 @@ class CachingHostnameResolver: def resolveHostName( self, resolutionReceiver, - hostName, + hostName: str, portNumber=0, addressTypes=None, transportSemantics="TCP", diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 58884f21a..9e411d4aa 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -5,6 +5,7 @@ based on different criteria. from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data +from typing import Dict, Mapping, Optional, Type, Union from scrapy.http import Response from scrapy.utils.misc import load_object @@ -29,15 +30,19 @@ class ResponseTypes: "text/*": "scrapy.http.TextResponse", } - def __init__(self): - self.classes = {} - self.mimetypes = MimeTypes() - mimedata = get_data("scrapy", "mime.types").decode("utf8") - self.mimetypes.readfp(StringIO(mimedata)) + def __init__(self) -> None: + self.classes: Dict[str, Type[Response]] = {} + self.mimetypes: MimeTypes = MimeTypes() + mimedata = get_data("scrapy", "mime.types") + if not mimedata: + raise ValueError( + "The mime.types file is not found in the Scrapy installation" + ) + self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype): + def from_mimetype(self, mimetype: str) -> Type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -46,7 +51,9 @@ class ResponseTypes: basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) - def from_content_type(self, content_type, content_encoding=None): + def from_content_type( + self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + ) -> Type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -56,7 +63,9 @@ class ResponseTypes: ) return self.from_mimetype(mimetype) - def from_content_disposition(self, content_disposition): + def from_content_disposition( + self, content_disposition: Union[str, bytes] + ) -> Type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -68,7 +77,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers): + def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -81,14 +90,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename): + def from_filename(self, filename: str) -> Type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body): + def from_body(self, body: bytes) -> Type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -106,7 +115,13 @@ class ResponseTypes: return self.from_mimetype("text/html") return self.from_mimetype("text") - def from_args(self, headers=None, url=None, filename=None, body=None): + def from_args( + self, + headers: Optional[Mapping[bytes, bytes]] = None, + url: Optional[str] = None, + filename: Optional[str] = None, + body: Optional[bytes] = None, + ) -> Type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 604b5e314..6ea2bfd97 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,9 +1,20 @@ +from __future__ import annotations + import logging import sys from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Union +from warnings import warn +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -29,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: """Parse the content of a robots.txt_ file as bytes. This must be a class method. It must return a new instance of the parser backend. @@ -42,14 +53,14 @@ class RobotParser(metaclass=ABCMeta): pass @abstractmethod - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: str + :type url: str or bytes :param user_agent: User agent - :type user_agent: str + :type user_agent: str or bytes """ pass @@ -79,6 +90,7 @@ class PythonRobotParser(RobotParser): class ReppyRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): + warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots self.spider = spider diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index caff79e9c..863fb6032 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,11 @@ """ XPath selectors based on lxml """ +from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response, st): +def _st(response: Optional[TextResponse], st: Optional[str]) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text, st): - rt = XmlResponse if st == "xml" else HtmlResponse +def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: + rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): + def __init__( + self, + response: Optional[TextResponse] = None, + text: Optional[str] = None, + type: Optional[str] = None, + root: Optional[Any] = _NOT_SET, + **kwargs: Any, + ): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index a3b849f7b..b5d8fdb12 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,21 +1,52 @@ +from __future__ import annotations + import copy import json -from collections.abc import MutableMapping from importlib import import_module from pprint import pformat +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Optional, + Tuple, + Union, + cast, +) from scrapy.settings import default_settings -SETTINGS_PRIORITIES = { +# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, +# see https://github.com/scrapy/scrapy/issues/5383. +_SettingsKeyT = Union[bool, float, int, str, None] + +if TYPE_CHECKING: + # https://github.com/python/typing/issues/445#issuecomment-1131458824 + from _typeshed import SupportsItems + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + + +SETTINGS_PRIORITIES: Dict[str, int] = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, } -def get_settings_priority(priority): +def get_settings_priority(priority: Union[int, str]) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -34,14 +65,15 @@ class SettingsAttribute: for settings configuration, not this one. """ - def __init__(self, value, priority): - self.value = value + def __init__(self, value: Any, priority: int): + self.value: Any = value + self.priority: int if isinstance(self.value, BaseSettings): self.priority = max(self.value.maxpriority(), priority) else: self.priority = priority - def set(self, value, priority): + def set(self, value: Any, priority: int) -> None: """Sets value if priority is higher or equal than current priority.""" if priority >= self.priority: if isinstance(self.value, BaseSettings): @@ -49,11 +81,11 @@ class SettingsAttribute: self.value = value self.priority = priority - def __repr__(self): + def __repr__(self) -> str: return f"" -class BaseSettings(MutableMapping): +class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -75,21 +107,25 @@ class BaseSettings(MutableMapping): highest priority will be retrieved. """ - def __init__(self, values=None, priority="project"): - self.frozen = False - self.attributes = {} + __default = object() + + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): + self.frozen: bool = False + self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name): + def __getitem__(self, opt_name: _SettingsKeyT) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value - def __contains__(self, name): + def __contains__(self, name: Any) -> bool: return name in self.attributes - def get(self, name, default=None): + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -101,7 +137,7 @@ class BaseSettings(MutableMapping): """ return self[name] if self[name] is not None else default - def getbool(self, name, default=False): + def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -131,7 +167,7 @@ class BaseSettings(MutableMapping): "'True'/'False' and 'true'/'false'" ) - def getint(self, name, default=0): + def getint(self, name: _SettingsKeyT, default: int = 0) -> int: """ Get a setting value as an int. @@ -143,7 +179,7 @@ class BaseSettings(MutableMapping): """ return int(self.get(name, default)) - def getfloat(self, name, default=0.0): + def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -155,7 +191,9 @@ class BaseSettings(MutableMapping): """ return float(self.get(name, default)) - def getlist(self, name, default=None): + def getlist( + self, name: _SettingsKeyT, default: Optional[List[Any]] = None + ) -> List[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -174,7 +212,9 @@ class BaseSettings(MutableMapping): value = value.split(",") return list(value) - def getdict(self, name, default=None): + def getdict( + self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None + ) -> Dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -195,7 +235,11 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def getdictorlist(self, name, default=None): + def getdictorlist( + self, + name: _SettingsKeyT, + default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, + ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -222,24 +266,31 @@ class BaseSettings(MutableMapping): return {} if isinstance(value, str): try: - return json.loads(value) + value_loaded = json.loads(value) + assert isinstance(value_loaded, (dict, list)) + return value_loaded except ValueError: return value.split(",") + if isinstance(value, tuple): + return list(value) + assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name): + def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": """Get a composition of a dictionary-like setting and its `_BASE` counterpart. :param name: name of the dictionary-like setting :type name: str """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") compbs = BaseSettings() compbs.update(self[name + "_BASE"]) compbs.update(self[name]) return compbs - def getpriority(self, name): + def getpriority(self, name: _SettingsKeyT) -> Optional[int]: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -251,7 +302,7 @@ class BaseSettings(MutableMapping): return None return self.attributes[name].priority - def maxpriority(self): + def maxpriority(self) -> int: """ Return the numerical value of the highest priority present throughout all settings, or the numerical value for ``default`` from @@ -259,13 +310,15 @@ class BaseSettings(MutableMapping): stored. """ if len(self) > 0: - return max(self.getpriority(name) for name in self) + return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") - def __setitem__(self, name, value): + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) - def set(self, name, value, priority="project"): + def set( + self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + ) -> None: """ Store a key/value attribute with a given priority. @@ -293,17 +346,26 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) - def setdefault(self, name, default=None, priority="project"): + def setdefault( + self, + name: _SettingsKeyT, + default: Any = None, + priority: Union[int, str] = "project", + ) -> Any: if name not in self: self.set(name, default, priority) return default return self.attributes[name].value - def setdict(self, values, priority="project"): + def setdict( + self, values: _SettingsInputT, priority: Union[int, str] = "project" + ) -> None: self.update(values, priority) - def setmodule(self, module, priority="project"): + def setmodule( + self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + ) -> None: """ Store settings from a module with a given priority. @@ -325,7 +387,8 @@ class BaseSettings(MutableMapping): if key.isupper(): self.set(key, getattr(module, key), priority) - def update(self, values, priority="project"): + # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports + def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -349,30 +412,34 @@ class BaseSettings(MutableMapping): """ self._assert_mutability() if isinstance(values, str): - values = json.loads(values) + values = cast(dict, json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, values.getpriority(name)) + self.set(name, value, cast(int, values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) - def delete(self, name, priority="project"): + def delete( + self, name: _SettingsKeyT, priority: Union[int, str] = "project" + ) -> None: + if name not in self: + raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= self.getpriority(name): + if priority >= cast(int, self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name): + def __delitem__(self, name: _SettingsKeyT) -> None: self._assert_mutability() del self.attributes[name] - def _assert_mutability(self): + def _assert_mutability(self) -> None: if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self): + def copy(self) -> "Self": """ Make a deep copy of current settings. @@ -384,7 +451,7 @@ class BaseSettings(MutableMapping): """ return copy.deepcopy(self) - def freeze(self): + def freeze(self) -> None: """ Disable further changes to the current settings. @@ -394,7 +461,7 @@ class BaseSettings(MutableMapping): """ self.frozen = True - def frozencopy(self): + def frozencopy(self) -> "Self": """ Return an immutable copy of the current settings. @@ -404,26 +471,26 @@ class BaseSettings(MutableMapping): copy.freeze() return copy - def __iter__(self): + def __iter__(self) -> Iterator[_SettingsKeyT]: return iter(self.attributes) - def __len__(self): + def __len__(self) -> int: return len(self.attributes) - def _to_dict(self): + def _to_dict(self) -> Dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value): + def _get_key(self, key_value: Any) -> _SettingsKeyT: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self): + def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -439,12 +506,25 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() - def _repr_pretty_(self, p, cycle): + # https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing + def _repr_pretty_(self, p: Any, cycle: bool) -> None: if cycle: p.text(repr(self)) else: p.text(pformat(self.copy_to_dict())) + def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: + try: + value = self.attributes[name].value + except KeyError: + if default is self.__default: + raise + + return default + else: + self.__delitem__(name) + return value + class Settings(BaseSettings): """ @@ -457,7 +537,9 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values=None, priority="project"): + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -471,15 +553,17 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings(): +def iter_default_settings() -> Iterable[Tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): yield name, getattr(default_settings, name) -def overridden_settings(settings): - """Return a dict of the settings that have been overridden""" +def overridden_settings( + settings: Mapping[_SettingsKeyT, Any] +) -> Iterable[Tuple[str, Any]]: + """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] if not isinstance(defvalue, dict) and value != defvalue: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 04bb5a696..ac945a1df 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,8 @@ import sys from importlib import import_module from pathlib import Path +ADDONS = {} + AJAXCRAWL_ENABLED = False ASYNCIO_EVENT_LOOP = None @@ -142,7 +144,7 @@ EXTENSIONS_BASE = { FEED_TEMPDIR = None FEEDS = {} FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = False +FEED_STORE_EMPTY = True FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None FEED_STORAGES = {} @@ -205,6 +207,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +JOBDIR = None + LOG_ENABLED = True LOG_ENCODING = "utf-8" LOG_FORMATTER = "scrapy.logformatter.LogFormatter" @@ -241,6 +245,10 @@ METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" +PERIODIC_LOG_DELTA = None +PERIODIC_LOG_STATS = None +PERIODIC_LOG_TIMING_ENABLED = False + RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 @@ -259,6 +267,21 @@ RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 +RETRY_EXCEPTIONS = [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.DNSLookupError", + "twisted.internet.error.ConnectionRefusedError", + "twisted.internet.error.ConnectionDone", + "twisted.internet.error.ConnectError", + "twisted.internet.error.ConnectionLost", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + # OSError is raised by the HttpCompression middleware when trying to + # decompress an empty response + OSError, + "scrapy.core.downloader.handlers.http11.TunnelError", +] ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 02a451a2b..d855c962c 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,13 +1,23 @@ +from __future__ import annotations + import traceback import warnings from collections import defaultdict +from types import ModuleType +from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer +from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader +from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(ISpiderLoader) class SpiderLoader: @@ -16,14 +26,14 @@ class SpiderLoader: in a Scrapy project. """ - def __init__(self, settings): - self.spider_modules = settings.getlist("SPIDER_MODULES") - self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders = {} - self._found = defaultdict(list) + def __init__(self, settings: BaseSettings): + self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") + self._spiders: Dict[str, Type[Spider]] = {} + self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() - def _check_name_duplicates(self): + def _check_name_duplicates(self) -> None: dupes = [] for name, locations in self._found.items(): dupes.extend( @@ -42,12 +52,12 @@ class SpiderLoader: category=UserWarning, ) - def _load_spiders(self, module): + def _load_spiders(self, module: ModuleType) -> None: for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): + def _load_all_spiders(self) -> None: for name in self.spider_modules: try: for module in walk_modules(name): @@ -65,10 +75,10 @@ class SpiderLoader: self._check_name_duplicates() @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name): + def load(self, spider_name: str) -> Type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -78,7 +88,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request): + def find_by_request(self, request: Request) -> List[str]: """ Return the list of spider names that can handle the given request. """ @@ -86,7 +96,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self): + def list(self) -> List[str]: """ Return a list with the names of all spiders available in the project. """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index eadc7c6ab..1e96654e2 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -4,46 +4,67 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations -from scrapy.http import Request +import logging +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable + +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.http import Request, Response +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class DepthMiddleware: - def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): + def __init__( + self, + maxdepth: int, + stats: StatsCollector, + verbose_stats: bool = False, + prio: int = 1, + ): self.maxdepth = maxdepth self.stats = stats self.verbose_stats = verbose_stats self.prio = prio @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: settings = crawler.settings maxdepth = settings.getint("DEPTH_LIMIT") verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") + assert crawler.stats return cls(maxdepth, crawler.stats, verbose, prio) - def process_spider_output(self, response, result, spider): + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: self._init_depth(response, spider) - return (r for r in result or () if self._filter(r, response, spider)) + return (r for r in result if self._filter(r, response, spider)) - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: self._init_depth(response, spider) - async for r in result or (): + async for r in result: if self._filter(r, response, spider): yield r - def _init_depth(self, response, spider): + def _init_depth(self, response: Response, spider: Spider) -> None: # base case (depth=0) if "depth" not in response.meta: response.meta["depth"] = 0 if self.verbose_stats: self.stats.inc_value("request_depth_count/0", spider=spider) - def _filter(self, request, response, spider): + def _filter(self, request: Any, response: Response, spider: Spider) -> bool: if not isinstance(request, Request): return True depth = response.meta["depth"] + 1 diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 0d3e5fe0b..94450b35b 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,9 +3,20 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations +import logging +from typing import TYPE_CHECKING, Any, Iterable, List, Optional + +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest +from scrapy.http import Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -13,21 +24,23 @@ logger = logging.getLogger(__name__) class HttpError(IgnoreRequest): """A non-200 response was filtered""" - def __init__(self, response, *args, **kwargs): + def __init__(self, response: Response, *args: Any, **kwargs: Any): self.response = response super().__init__(*args, **kwargs) class HttpErrorMiddleware: @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def __init__(self, settings): - self.handle_httpstatus_all = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list = settings.getlist("HTTPERROR_ALLOWED_CODES") + def __init__(self, settings: BaseSettings): + self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") + self.handle_httpstatus_list: List[int] = settings.getlist( + "HTTPERROR_ALLOWED_CODES" + ) - def process_spider_input(self, response, spider): + def process_spider_input(self, response: Response, spider: Spider) -> None: if 200 <= response.status < 300: # common case return meta = response.meta @@ -45,8 +58,11 @@ class HttpErrorMiddleware: return raise HttpError(response, "Ignoring non-200 response") - def process_spider_exception(self, response, exception, spider): + def process_spider_exception( + self, response: Response, exception: Exception, spider: Spider + ) -> Optional[Iterable[Any]]: if isinstance(exception, HttpError): + assert spider.crawler.stats spider.crawler.stats.inc_value("httperror/response_ignored_count") spider.crawler.stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" @@ -57,3 +73,4 @@ class HttpErrorMiddleware: extra={"spider": spider}, ) return [] + return None diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 1a48926b3..a5214702d 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,36 +3,51 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ +from __future__ import annotations + import logging import re import warnings +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set -from scrapy import signals -from scrapy.http import Request +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Request, Response +from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class OffsiteMiddleware: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def process_spider_output(self, response, result, spider): - return (r for r in result or () if self._filter(r, spider)) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (r for r in result if self._filter(r, spider)) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: if self._filter(r, spider): yield r - def _filter(self, request, spider) -> bool: + def _filter(self, request: Any, spider: Spider) -> bool: if not isinstance(request, Request): return True if request.dont_filter or self.should_follow(request, spider): @@ -49,13 +64,13 @@ class OffsiteMiddleware: self.stats.inc_value("offsite/filtered", spider=spider) return False - def should_follow(self, request, spider): + def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) - def get_host_regex(self, spider): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: @@ -83,9 +98,9 @@ class OffsiteMiddleware: regex = rf'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) - def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) - self.domains_seen = set() + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) + self.domains_seen: Set[str] = set() class URLWarning(Warning): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index fd91e658b..a29e0ebb5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,20 +2,39 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from __future__ import annotations + import warnings -from typing import Tuple +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Dict, + Iterable, + Optional, + Tuple, + Type, + Union, + cast, +) from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url -LOCAL_SCHEMES = ( +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", "data", @@ -37,18 +56,20 @@ class ReferrerPolicy: NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES name: str - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: raise NotImplementedError() - def stripped_referrer(self, url): + def stripped_referrer(self, url: str) -> Optional[str]: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.strip_url(url) + return None - def origin_referrer(self, url): + def origin_referrer(self, url: str) -> Optional[str]: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.origin(url) + return None - def strip_url(self, url, origin_only=False): + def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]: """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -72,18 +93,18 @@ class ReferrerPolicy: origin_only=origin_only, ) - def origin(self, url): + def origin(self, url: str) -> Optional[str]: """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(url, origin_only=True) - def potentially_trustworthy(self, url): + def potentially_trustworthy(self, url: str) -> bool: # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy parsed_url = urlparse(url) if parsed_url.scheme in ("data",): return False return self.tls_protected(url) - def tls_protected(self, url): + def tls_protected(self, url: str) -> bool: return urlparse(url).scheme in ("https", "ftps") @@ -98,7 +119,7 @@ class NoReferrerPolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return None @@ -119,9 +140,10 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if not self.tls_protected(response_url) or self.tls_protected(request_url): return self.stripped_referrer(response_url) + return None class SameOriginPolicy(ReferrerPolicy): @@ -137,9 +159,10 @@ class SameOriginPolicy(ReferrerPolicy): name: str = POLICY_SAME_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if self.origin(response_url) == self.origin(request_url): return self.stripped_referrer(response_url) + return None class OriginPolicy(ReferrerPolicy): @@ -154,7 +177,7 @@ class OriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return self.origin_referrer(response_url) @@ -174,13 +197,14 @@ class StrictOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) + return None class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -197,7 +221,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -224,7 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -234,6 +258,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) + return None class UnsafeUrlPolicy(ReferrerPolicy): @@ -252,7 +277,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name: str = POLICY_UNSAFE_URL - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return self.stripped_referrer(response_url) @@ -267,7 +292,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): name: str = POLICY_SCRAPY_DEFAULT -_policy_classes = { +_policy_classes: Dict[str, Type[ReferrerPolicy]] = { p.name: p for p in ( NoReferrerPolicy, @@ -286,14 +311,16 @@ _policy_classes = { _policy_classes[""] = NoReferrerWhenDowngradePolicy -def _load_policy_class(policy, warning_only=False): +def _load_policy_class( + policy: str, warning_only: bool = False +) -> Optional[Type[ReferrerPolicy]]: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return load_object(policy) + return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: try: return _policy_classes[policy.lower()] @@ -307,13 +334,15 @@ def _load_policy_class(policy, warning_only=False): class RefererMiddleware: - def __init__(self, settings=None): - self.default_policy = DefaultReferrerPolicy + def __init__(self, settings: Optional[BaseSettings] = None): + self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: - self.default_policy = _load_policy_class(settings.get("REFERRER_POLICY")) + settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) + assert settings_policy + self.default_policy = settings_policy @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("REFERER_ENABLED"): raise NotConfigured mw = cls(crawler.settings) @@ -323,7 +352,9 @@ class RefererMiddleware: return mw - def policy(self, resp_or_url, request): + def policy( + self, resp_or_url: Union[Response, str], request: Request + ) -> ReferrerPolicy: """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. @@ -348,21 +379,25 @@ class RefererMiddleware: cls = _load_policy_class(policy_name, warning_only=True) return cls() if cls else self.default_policy() - def process_spider_output(self, response, result, spider): - return (self._set_referer(r, response) for r in result or ()) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (self._set_referer(r, response) for r in result) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: yield self._set_referer(r, response) - def _set_referer(self, r, response): + def _set_referer(self, r: Any, response: Response) -> Any: if isinstance(r, Request): referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault("Referer", referrer) return r - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: # check redirected request to patch "Referer" header if necessary redirected_urls = request.meta.get("redirect_urls", []) if redirected_urls: @@ -378,7 +413,7 @@ class RefererMiddleware: policy_referrer = self.policy(parent_url, request).referrer( parent_url, request.url ) - if policy_referrer != request_referrer: + if policy_referrer != request_referrer.decode("latin1"): if policy_referrer is None: request.headers.pop("Referer") else: diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f6d92e53a..e2aa554a7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -4,40 +4,54 @@ Url Length Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations +import logging +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable + +from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): - self.maxlength = maxlength + def __init__(self, maxlength: int): + self.maxlength: int = maxlength @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: maxlength = settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured return cls(maxlength) - def process_spider_output(self, response, result, spider): - return (r for r in result or () if self._filter(r, spider)) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (r for r in result if self._filter(r, spider)) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: if self._filter(r, spider): yield r - def _filter(self, request, spider): + def _filter(self, request: Any, spider: Spider) -> bool: if isinstance(request, Request) and len(request.url) > self.maxlength: logger.info( "Ignoring link (url length > %(maxlength)d): %(url)s ", {"maxlength": self.maxlength, "url": request.url}, extra={"spider": spider}, ) + assert spider.crawler.stats spider.crawler.stats.inc_value( "urllength/request_ignored_count", spider=spider ) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 3502f8b27..e16d71727 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast + +from twisted.internet.defer import Deferred from scrapy import signals -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class Spider(object_ref): @@ -25,21 +31,21 @@ class Spider(object_ref): name: str custom_settings: Optional[dict] = None - def __init__(self, name=None, **kwargs): + def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: self.name = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls = [] + self.start_urls: List[str] = [] @property - def logger(self): + def logger(self) -> logging.LoggerAdapter: logger = logging.getLogger(self.name) return logging.LoggerAdapter(logger, {"spider": self}) - def log(self, message, level=logging.DEBUG, **kw): + def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level This helper wraps a log call to the logger within the spider, but you @@ -49,17 +55,17 @@ class Spider(object_ref): self.logger.log(level, message, **kw) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = cls(*args, **kwargs) spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler: Crawler): + def _set_crawler(self, crawler: Crawler) -> None: self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self): + def start_requests(self) -> Iterable[Request]: if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " @@ -69,29 +75,30 @@ class Spider(object_ref): for url in self.start_urls: yield Request(url, dont_filter=True) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response, **kwargs): + def parse(self, response: Response, **kwargs: Any) -> Any: raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) @classmethod - def update_settings(cls, settings): + def update_settings(cls, settings: BaseSettings) -> None: settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod - def handles_request(cls, request): + def handles_request(cls, request: Request) -> bool: return url_is_from_spider(request.url, cls) @staticmethod - def close(spider, reason): + def close(spider: Spider, reason: str) -> Union[Deferred, None]: closed = getattr(spider, "closed", None) if callable(closed): - return closed(reason) + return cast(Union[Deferred, None], closed(reason)) + return None - def __repr__(self): + def __repr__(self) -> str: return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6afe0d636..f665ad88c 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -10,7 +10,6 @@ from typing import Union from queuelib import queue -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.request import request_from_dict @@ -143,32 +142,3 @@ MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDisk MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) - - -# deprecated queue classes -_subclass_warn_message = "{cls} inherits from deprecated class {old}" -_instance_warn_message = "{cls} is deprecated" -PickleFifoDiskQueueNonRequest = create_deprecated_class( - name="PickleFifoDiskQueueNonRequest", - new_class=_PickleFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -PickleLifoDiskQueueNonRequest = create_deprecated_class( - name="PickleLifoDiskQueueNonRequest", - new_class=_PickleLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalFifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalFifoDiskQueueNonRequest", - new_class=_MarshalFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalLifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalLifoDiskQueueNonRequest", - new_class=_MarshalLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..53cfeddd0 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,7 @@ """Boto/botocore helpers""" -def is_botocore_available(): +def is_botocore_available() -> bool: try: import botocore # noqa: F401 diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..641dfa4a2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,18 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Union, +) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -13,21 +24,26 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, custom=None, convert=update_classpath): +def build_component_list( + compdict: MutableMapping[Any, Any], + custom: Any = None, + convert: Callable[[Any], Any] = update_classpath, +) -> List[Any]: """Compose a component list from a { class: order } dictionary.""" - def _check_components(complist): + def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): raise ValueError( f"Some paths in {complist!r} convert to the same object, " "please update your settings" ) - def _map_keys(compdict): + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): prio = compdict.getpriority(k) + assert prio is not None if compbs.getpriority(convert(k)) == prio: raise ValueError( f"Some paths in {list(compdict.keys())!r} " @@ -40,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in compdict.items()} - def _validate_values(compdict): + def _validate_values(compdict: Mapping[Any, Any]) -> None: """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): @@ -49,11 +65,17 @@ def build_component_list(compdict, custom=None, convert=update_classpath): "please provide a real number or None instead" ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) - if custom is not None: + warnings.warn( + "The 'custom' attribute of build_component_list() is deprecated. " + "Please merge its value into 'compdict' manually or change your " + "code to use Settings.getwithbase().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) # type: ignore[return-value] compdict.update(custom) _validate_values(compdict) @@ -61,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist): +def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -84,7 +106,7 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def init_env(project="default", set_syspath=True): +def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to be able to locate the project module. @@ -99,7 +121,7 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): +def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) cfg = ConfigParser() @@ -107,7 +129,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> List[str]: +def get_sources(use_closest: bool = True) -> List[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -122,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]: return sources -def feed_complete_default_values_from_settings(feed, settings): +def feed_complete_default_values_from_settings( + feed: Dict[str, Any], settings: BaseSettings +) -> Dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -138,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings): def feed_process_params_from_cli( - settings, + settings: BaseSettings, output: List[str], - output_format=None, + output_format: Optional[str] = None, overwrite_output: Optional[List[str]] = None, -): +) -> Dict[str, Dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ - valid_output_formats = without_none_values( + valid_output_formats: Iterable[str] = without_none_values( settings.getwithbase("FEED_EXPORTERS") ).keys() - def check_valid_format(output_format): + def check_valid_format(output_format: str) -> None: if output_format not in valid_output_formats: raise UsageError( f"Unrecognized output format '{output_format}'. " @@ -202,7 +226,8 @@ def feed_process_params_from_cli( for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) - except ValueError: + check_valid_format(feed_format) + except (ValueError, UsageError): feed_uri = element feed_format = Path(element).suffix.replace(".", "") else: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index fa57a4f26..0ba2fe4e2 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,76 +5,159 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ +from __future__ import annotations + import collections +import warnings import weakref from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Iterable, + Optional, + OrderedDict, + Sequence, + Tuple, + TypeVar, + Union, +) + +from scrapy.exceptions import ScrapyDeprecationWarning + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") class CaselessDict(dict): __slots__ = () - def __init__(self, seq=None): + def __new__(cls, *args: Any, **kwargs: Any) -> Self: + from scrapy.http.headers import Headers + + if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): + warnings.warn( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, *args, **kwargs) + + def __init__( + self, + seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + ): super().__init__() if seq: self.update(seq) - def __getitem__(self, key): + def __getitem__(self, key: AnyStr) -> Any: return dict.__getitem__(self, self.normkey(key)) - def __setitem__(self, key, value): + def __setitem__(self, key: AnyStr, value: Any) -> None: dict.__setitem__(self, self.normkey(key), self.normvalue(value)) - def __delitem__(self, key): + def __delitem__(self, key: AnyStr) -> None: dict.__delitem__(self, self.normkey(key)) - def __contains__(self, key): + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] return dict.__contains__(self, self.normkey(key)) has_key = __contains__ - def __copy__(self): + def __copy__(self) -> Self: return self.__class__(self) copy = __copy__ - def normkey(self, key): + def normkey(self, key: AnyStr) -> AnyStr: """Method to normalize dictionary key access""" return key.lower() - def normvalue(self, value): + def normvalue(self, value: Any) -> Any: """Method to normalize values prior to be set""" return value - def get(self, key, def_val=None): + def get(self, key: AnyStr, def_val: Any = None) -> Any: return dict.get(self, self.normkey(key), self.normvalue(def_val)) - def setdefault(self, key, def_val=None): - return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) + def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: + return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] - def update(self, seq): + # doesn't fully implement MutableMapping.update() + def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @classmethod - def fromkeys(cls, keys, value=None): - return cls((k, value) for k in keys) + def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override] + return cls((k, value) for k in keys) # type: ignore[misc] - def pop(self, key, *args): + def pop(self, key: AnyStr, *args: Any) -> Any: return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict): +class CaseInsensitiveDict(collections.UserDict): + """A dict-like structure that accepts strings or bytes + as keys and allows case-insensitive lookups. + """ + + def __init__(self, *args, **kwargs) -> None: + self._keys: dict = {} + super().__init__(*args, **kwargs) + + def __getitem__(self, key: AnyStr) -> Any: + normalized_key = self._normkey(key) + return super().__getitem__(self._keys[normalized_key.lower()]) + + def __setitem__(self, key: AnyStr, value: Any) -> None: + normalized_key = self._normkey(key) + try: + lower_key = self._keys[normalized_key.lower()] + del self[lower_key] + except KeyError: + pass + super().__setitem__(normalized_key, self._normvalue(value)) + self._keys[normalized_key.lower()] = normalized_key + + def __delitem__(self, key: AnyStr) -> None: + normalized_key = self._normkey(key) + stored_key = self._keys.pop(normalized_key.lower()) + super().__delitem__(stored_key) + + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + normalized_key = self._normkey(key) + return normalized_key.lower() in self._keys + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {super().__repr__()}>" + + def _normkey(self, key: AnyStr) -> AnyStr: + return key + + def _normvalue(self, value: Any) -> Any: + return value + + +class LocalCache(OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.limit = limit + self.limit: Optional[int] = limit - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: while len(self) >= self.limit: self.popitem(last=False) @@ -93,17 +176,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.data = LocalCache(limit=limit) + self.data: LocalCache = LocalCache(limit=limit) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: try: super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): @@ -113,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq): - self.seq = seq + def __init__(self, seq: Sequence): + self.seq: Sequence = seq - def __contains__(self, item): + def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4e684645b..04186559f 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,19 +1,21 @@ import warnings from functools import wraps +from typing import Any, Callable from twisted.internet import defer, threads +from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning -def deprecated(use_instead=None): +def deprecated(use_instead: Any = None) -> Callable: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func): + def deco(func: Callable) -> Callable: @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: Any, **kwargs: Any) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -28,23 +30,23 @@ def deprecated(use_instead=None): return deco -def defers(func): +def defers(func: Callable) -> Callable[..., Deferred]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func): +def inthread(func: Callable) -> Callable[..., Deferred]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a46274fef..bf3c5ef5b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,19 +5,26 @@ import asyncio import inspect from asyncio import Future from functools import wraps +from types import CoroutineType from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Awaitable, Callable, Coroutine, + Dict, Generator, Iterable, Iterator, List, Optional, + Tuple, + TypeVar, Union, cast, + overload, ) from twisted.internet import defer @@ -44,7 +51,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result) -> Deferred: +def defer_succeed(result: Any) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -58,7 +65,7 @@ def defer_succeed(result) -> Deferred: return d -def defer_result(result) -> Deferred: +def defer_result(result: Any) -> Deferred: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -66,7 +73,7 @@ def defer_result(result) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: +def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -84,7 +91,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args, **named + iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -146,14 +153,14 @@ class _AsyncCooperatorAdapter(Iterator): self, aiterable: AsyncIterable, callable: Callable, - *callable_args, - **callable_kwargs + *callable_args: Any, + **callable_kwargs: Any, ): - self.aiterator = aiterable.__aiter__() - self.callable = callable - self.callable_args = callable_args - self.callable_kwargs = callable_kwargs - self.finished = False + self.aiterator: AsyncIterator = aiterable.__aiter__() + self.callable: Callable = callable + self.callable_args: Tuple[Any, ...] = callable_args + self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] self.anext_deferred: Optional[Deferred] = None @@ -183,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = cast( - Deferred, deferred_from_coro(self.aiterator.__anext__()) - ) + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -201,7 +206,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named + async_iterable: AsyncIterable, + count: int, + callable: Callable, + *args: Any, + **named: Any, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -210,7 +219,9 @@ def parallel_async( return dl -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_chain( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() for x in callbacks: @@ -220,7 +231,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: def process_chain_both( - callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw + callbacks: Iterable[Callable], + errbacks: Iterable[Callable], + input: Any, + *a: Any, + **kw: Any, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() @@ -240,7 +255,9 @@ def process_chain_both( return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_parallel( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -250,7 +267,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: +def iter_errback( + iterable: Iterable, errback: Callable, *a: Any, **kw: Any +) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -265,7 +284,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a, **kw + aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any ) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() @@ -280,7 +299,21 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o) -> Any: +_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +_T = TypeVar("_T") + + +@overload +def deferred_from_coro(o: _CT) -> Deferred: + ... + + +@overload +def deferred_from_coro(o: _T) -> _T: + ... + + +def deferred_from_coro(o: _T) -> Union[Deferred, _T]: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -303,13 +336,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args, **coro_kwargs): + def f(*coro_args: Any, **coro_kwargs: Any) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: +def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f4d6e0451..ea577c44a 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,12 +2,12 @@ import inspect import warnings -from typing import List, Tuple +from typing import Any, Dict, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning -def attribute(obj, oldattr, newattr, version="0.12"): +def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: cname = obj.__class__.__name__ warnings.warn( f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " @@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"): def create_deprecated_class( - name, - new_class, - clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", - instance_warn_message="{cls} is deprecated, instantiate {new} instead.", -): + name: str, + new_class: type, + clsdict: Optional[Dict[str, Any]] = None, + warn_category: Type[Warning] = ScrapyDeprecationWarning, + warn_once: bool = True, + old_class_path: Optional[str] = None, + new_class_path: Optional[str] = None, + subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", +) -> type: """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. @@ -53,17 +53,20 @@ def create_deprecated_class( OldName. """ - class DeprecatedClass(new_class.__class__): - deprecated_class = None - warned_on_subclass = False + # https://github.com/python/mypy/issues/4177 + class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + deprecated_class: Optional[type] = None + warned_on_subclass: bool = False - def __new__(metacls, name, bases, clsdict_): + def __new__( + metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name, bases, clsdict_): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -81,10 +84,10 @@ def create_deprecated_class( # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details - def __instancecheck__(cls, inst): + def __instancecheck__(cls, inst: Any) -> bool: return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) - def __subclasscheck__(cls, sub): + def __subclasscheck__(cls, sub: type) -> bool: if cls is not DeprecatedClass.deprecated_class: # we should do the magic only if second `issubclass` argument # is the deprecated class itself - subclasses of the @@ -98,7 +101,7 @@ def create_deprecated_class( mro = getattr(sub, "__mro__", ()) return any(c in {cls, new_class} for c in mro) - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: old = DeprecatedClass.deprecated_class if cls is old: msg = instance_warn_message.format( @@ -125,7 +128,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls, forced=None): +def _clspath(cls: type, forced: Optional[str] = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" @@ -134,7 +137,17 @@ def _clspath(cls, forced=None): DEPRECATION_RULES: List[Tuple[str, str]] = [] -def update_classpath(path): +@overload +def update_classpath(path: str) -> str: + ... + + +@overload +def update_classpath(path: Any) -> Any: + ... + + +def update_classpath(path: Any) -> Any: """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: if isinstance(path, str) and path.startswith(prefix): @@ -147,7 +160,7 @@ def update_classpath(path): return path -def method_is_overridden(subclass, base_class, method_name): +def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool: """ Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 77c32b002..596cf89e4 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,17 +6,18 @@ import ctypes import platform import sys from pprint import pformat as pformat_ +from typing import Any from packaging.version import Version as parse_version -def _enable_windows_terminal_processing(): +def _enable_windows_terminal_processing() -> bool: # https://stackoverflow.com/a/36760881 - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) -def _tty_supports_color(): +def _tty_supports_color() -> bool: if sys.platform != "win32": return True @@ -28,7 +29,7 @@ def _tty_supports_color(): return _enable_windows_terminal_processing() -def _colorize(text, colorize=True): +def _colorize(text: str, colorize: bool = True) -> str: if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: @@ -42,9 +43,9 @@ def _colorize(text, colorize=True): return highlight(text, PythonLexer(), TerminalFormatter()) -def pformat(obj, *args, **kwargs): +def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: return _colorize(pformat_(obj), kwargs.pop("colorize", True)) -def pprint(obj, *args, **kwargs): +def pprint(obj: Any, *args: Any, **kwargs: Any) -> None: print(pformat(obj, *args, **kwargs)) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 8e3ec2c37..a5f2a8c6e 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,9 +2,13 @@ # used in global tests code from time import time # noqa: F401 +from typing import TYPE_CHECKING, Any, List, Tuple + +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine): +def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -23,7 +27,7 @@ def get_engine_status(engine): "engine.scraper.slot.needs_backout()", ] - checks = [] + checks: List[Tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] @@ -33,7 +37,7 @@ def get_engine_status(engine): return checks -def format_engine_status(engine=None): +def format_engine_status(engine: "ExecutionEngine") -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -43,5 +47,5 @@ def format_engine_status(engine=None): return s -def print_engine_status(engine): +def print_engine_status(engine: "ExecutionEngine") -> None: print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6bf6e9195..c77681a53 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,9 +1,10 @@ import posixpath from ftplib import FTP, error_perm from posixpath import dirname +from typing import IO -def ftp_makedirs_cwd(ftp, path, first_call=True): +def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. @@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( - *, path, file, host, port, username, password, use_active_mode=False, overwrite=True -): + *, + path: str, + file: IO, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, + overwrite: bool = True, +) -> None: """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..c7f74030e 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,34 +1,32 @@ import struct from gzip import GzipFile from io import BytesIO +from typing import List + +from scrapy.http import Response -def gunzip(data): +def gunzip(data: bytes) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] + output_list: List[bytes] = [] chunk = b"." while chunk: try: chunk = f.read1(8196) output_list.append(chunk) - except (IOError, EOFError, struct.error): + except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty and f.extrabuf - # contains the whole page content - if output_list or getattr(f, "extrabuf", None): - try: - output_list.append(f.extrabuf[-f.extrasize :]) - finally: - break - else: - raise + # some pages are quite small so output_list is empty + if output_list: + break + raise return b"".join(output_list) -def gzip_magic_number(response): +def gzip_magic_number(response: Response) -> bool: return response.body[:3] == b"\x1f\x8b\x08" diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 170055d5e..55362efdf 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -2,15 +2,34 @@ import csv import logging import re from io import StringIO +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Literal, + Optional, + Union, + cast, + overload, +) from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode +if TYPE_CHECKING: + from lxml._types import SupportsReadClose + logger = logging.getLogger(__name__) -def xmliter(obj, nodename): +def xmliter( + obj: Union[Response, str, bytes], nodename: str +) -> Generator[Selector, Any, None]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -27,20 +46,22 @@ def xmliter(obj, nodename): NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) - document_header = re.search(DOCUMENT_HEADER_RE, text) - document_header = document_header.group().strip() if document_header else "" + document_header_match = re.search(DOCUMENT_HEADER_RE, text) + document_header = ( + document_header_match.group().strip() if document_header_match else "" + ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces = {} + namespaces: Dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): + assert header_end_idx tag = re.search( rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: - namespaces.update( - reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()) - ) + for x in re.findall(NAMESPACE_RE, tag.group()): + namespaces[x[1]] = x[0] r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): @@ -54,12 +75,19 @@ def xmliter(obj, nodename): yield Selector(text=nodetext, type="xml") -def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): +def xmliter_lxml( + obj: Union[Response, str, bytes], + nodename: str, + namespace: Optional[str] = None, + prefix: str = "x", +) -> Generator[Selector, Any, None]: from lxml import etree reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) + iterable = etree.iterparse( + cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding="unicode") @@ -71,30 +99,46 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): class _StreamReader: - def __init__(self, obj): - self._ptr = 0 - if isinstance(obj, Response): + def __init__(self, obj: Union[Response, str, bytes]): + self._ptr: int = 0 + self._text: Union[str, bytes] + if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding + elif isinstance(obj, Response): + self._text, self.encoding = obj.body, "utf-8" else: self._text, self.encoding = obj, "utf-8" - self._is_unicode = isinstance(self._text, str) + self._is_unicode: bool = isinstance(self._text, str) + self._is_first_read: bool = True - def read(self, n=65535): - self.read = self._read_unicode if self._is_unicode else self._read_string - return self.read(n).lstrip() + def read(self, n: int = 65535) -> bytes: + method: Callable[[int], bytes] = ( + self._read_unicode if self._is_unicode else self._read_string + ) + result = method(n) + if self._is_first_read: + self._is_first_read = False + result = result.lstrip() + return result - def _read_string(self, n=65535): + def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e] + return cast(bytes, self._text)[s:e] - def _read_unicode(self, n=65535): + def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e].encode("utf-8") + return cast(str, self._text)[s:e].encode("utf-8") -def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): +def csviter( + obj: Union[Response, str, bytes], + delimiter: Optional[str] = None, + headers: Optional[List[str]] = None, + encoding: Optional[str] = None, + quotechar: Optional[str] = None, +) -> Generator[Dict[str, str], Any, None]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -112,12 +156,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - def row_to_unicode(row_): + def row_to_unicode(row_: Iterable) -> List[str]: return [to_unicode(field, encoding) for field in row_] lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs = {} + kwargs: Dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: @@ -147,7 +191,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): yield dict(zip(headers, row)) -def _body_or_str(obj, unicode=True): +@overload +def _body_or_str(obj: Union[Response, str, bytes]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: + ... + + +def _body_or_str( + obj: Union[Response, str, bytes], unicode: bool = True +) -> Union[str, bytes]: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 858affc03..e230e4235 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,9 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path = settings["JOBDIR"] - if path and not Path(path).exists(): + path: Optional[str] = settings["JOBDIR"] + if not path: + return None + if not Path(path).exists(): Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2ce4725f4..2a38f151a 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,24 +1,37 @@ +from __future__ import annotations + import logging import sys -import warnings from logging.config import dictConfig -from typing import Tuple +from types import TracebackType +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def failure_to_exc_info(failure: Failure): +def failure_to_exc_info( + failure: Failure, +) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): - return (failure.type, failure.value, failure.getTracebackObject()) + assert failure.type + assert failure.value + return ( + failure.type, + failure.value, + cast(Optional[TracebackType], failure.getTracebackObject()), + ) + return None class TopLevelFormatter(logging.Filter): @@ -33,10 +46,10 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers=None): - self.loggers = loggers or [] + def __init__(self, loggers: Optional[List[str]] = None): + self.loggers: List[str] = loggers or [] - def filter(self, record): + def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): record.name = record.name.split(".", 1)[0] return True @@ -62,7 +75,9 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None, install_root_handler=True): +def configure_logging( + settings: Union[Settings, dict, None] = None, install_root_handler: bool = True +) -> None: """ Initialize logging defaults for Scrapy. @@ -99,13 +114,16 @@ def configure_logging(settings=None, install_root_handler=True): settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) + sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] if install_root_handler: install_scrapy_root_handler(settings) -def install_scrapy_root_handler(settings): +_scrapy_root_handler: Optional[logging.Handler] = None + + +def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler if ( @@ -118,16 +136,14 @@ def install_scrapy_root_handler(settings): logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler(): +def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler = None - - -def _get_handler(settings): +def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") + handler: logging.Handler if filename: mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w" encoding = settings.get("LOG_ENCODING") @@ -181,16 +197,16 @@ class StreamLogger: https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ - def __init__(self, logger, log_level=logging.INFO): - self.logger = logger - self.log_level = log_level - self.linebuf = "" + def __init__(self, logger: logging.Logger, log_level: int = logging.INFO): + self.logger: logging.Logger = logger + self.log_level: int = log_level + self.linebuf: str = "" - def write(self, buf): + def write(self, buf: str) -> None: for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) - def flush(self): + def flush(self) -> None: for h in self.logger.handlers: h.flush() @@ -198,12 +214,13 @@ class StreamLogger: class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" - def __init__(self, crawler, *args, **kwargs): + def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.crawler = crawler + self.crawler: Crawler = crawler - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" + assert self.crawler.stats self.crawler.stats.inc_value(sname) @@ -213,18 +230,9 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: and adapts it into a tuple of positional arguments for logger.log calls, handling backward compatibility as well. """ - if not {"level", "msg", "args"} <= set(logkws): - warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning) - - if "format" in logkws: - warnings.warn( - "`format` key in LogFormatter methods has been " - "deprecated, use `msg` instead", - ScrapyDeprecationWarning, - ) level = logkws.get("level", logging.INFO) - message = logkws.get("format", logkws.get("msg")) + message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls args = logkws if not logkws.get("args") else logkws["args"] diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..a9364bea2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,14 +10,23 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import TYPE_CHECKING, Any, Callable, Union - -from w3lib.html import replace_entities +from types import ModuleType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Deque, + Generator, + Iterable, + List, + Optional, + Union, + cast, +) from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.python import flatten, to_unicode if TYPE_CHECKING: from scrapy import Spider @@ -26,7 +35,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -def arg_to_iter(arg): +def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -35,7 +44,7 @@ def arg_to_iter(arg): if arg is None: return [] if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return arg + return cast(Iterable[Any], arg) return [arg] @@ -53,7 +62,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: @@ -72,7 +81,7 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def walk_modules(path): +def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -80,7 +89,7 @@ def walk_modules(path): For example: walk_modules('scrapy.utils') """ - mods = [] + mods: List[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -94,36 +103,7 @@ def walk_modules(path): return mods -def extract_regex(regex, text, encoding="utf-8"): - """Extract a list of unicode strings from the given text/encoding using the following policies: - - * if the regex contains a named group called "extract" that will be returned - * if the regex contains multiple numbered groups, all those will be returned (flattened) - * if the regex doesn't contain any group the entire regex matching is returned - """ - warnings.warn( - "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - if isinstance(regex, str): - regex = re.compile(regex, re.UNICODE) - - try: - strings = [regex.search(text).group("extract")] # named group - except Exception: - strings = regex.findall(text) # full regex or numbered groups - strings = flatten(strings) - - if isinstance(text, str): - return [replace_entities(s, keep=["lt", "amp"]) for s in strings] - return [ - replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings - ] - - -def md5sum(file): +def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -140,7 +120,7 @@ def md5sum(file): return m.hexdigest() -def rel_has_nofollow(rel): +def rel_has_nofollow(rel: Optional[str]) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() @@ -181,7 +161,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): @contextmanager -def set_environ(**kwargs): +def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -198,11 +178,11 @@ def set_environ(**kwargs): os.environ[k] = v -def walk_callable(node): +def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo = deque([node]) + todo: Deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() @@ -217,15 +197,15 @@ def walk_callable(node): _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable): +def is_generator_with_return_value(callable: Callable) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise """ if callable in _generator_callbacks_cache: - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node): + def returns_none(return_node: ast.Return) -> bool: value = return_node.value return ( value is None or isinstance(value, ast.NameConstant) and value.value is None @@ -248,10 +228,10 @@ def is_generator_with_return_value(callable): for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) _generator_callbacks_cache[callable] = False - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 7646264a8..db9a71273 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,6 +1,13 @@ import signal +from types import FrameType +from typing import Any, Callable, Dict, Optional, Union -signal_names = {} +# copy of _HANDLER from typeshed/stdlib/signal.pyi +SignalHandlerT = Union[ + Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None +] + +signal_names: Dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) @@ -8,15 +15,14 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function, override_sigint=True): +def install_shutdown_handlers( + function: SignalHandlerT, override_sigint: bool = True +) -> None: """Install the given function as a signal handler for all common shutdown - signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the - SIGINT handler won't be install if there is already a handler in place - (e.g. Pdb) + signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the + SIGINT handler won't be installed if there is already a handler in place + (e.g. Pdb) """ - from twisted.internet import reactor - - reactor._handleSignals() signal.signal(signal.SIGTERM, function) if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index ab1b8e3ee..de3c8eaf9 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,9 @@ import os import warnings from importlib import import_module +from os import PathLike from pathlib import Path +from typing import Union from scrapy.exceptions import NotConfigured from scrapy.settings import Settings @@ -11,9 +13,9 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" -def inside_project(): - scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE") - if scrapy_module is not None: +def inside_project() -> bool: + scrapy_module = os.environ.get(ENVVAR) + if scrapy_module: try: import_module(scrapy_module) except ImportError as exc: @@ -25,7 +27,7 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project="default") -> str: +def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") @@ -44,7 +46,7 @@ def project_data_dir(project="default") -> str: return str(d) -def data_path(path: str, createdir=False) -> str: +def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. @@ -60,7 +62,7 @@ def data_path(path: str, createdir=False) -> str: return str(path_obj) -def get_project_settings(): +def get_project_settings() -> Settings: if ENVVAR not in os.environ: project = os.environ.get("SCRAPY_PROJECT", "default") init_env(project) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..0b5dc324f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import collections.abc import gc import inspect import re @@ -12,9 +13,17 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Callable, + Dict, + Generator, Iterable, + Iterator, + List, Mapping, Optional, + Pattern, + Tuple, Union, overload, ) @@ -22,7 +31,7 @@ from typing import ( from scrapy.utils.asyncgen import as_async_generator -def flatten(x): +def flatten(x: Iterable) -> list: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -42,7 +51,7 @@ def flatten(x): return list(iflatten(x)) -def iflatten(x): +def iflatten(x: Iterable) -> Iterable: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" @@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_, key=lambda x: x): +def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: """efficient function to uniquify a list preserving item order""" seen = set() result = [] @@ -124,7 +133,9 @@ def to_bytes( return text.encode(encoding, errors) -def re_rsearch(pattern, text, chunk_size=1024): +def re_rsearch( + pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 +) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024): the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter(): + def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024): return None -def memoizemethod_noargs(method): +def memoizemethod_noargs(method: Callable) -> Callable: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self, *args, **kwargs): + def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func, stripself=False): +def get_func_args(func: Callable, stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args = [] + args: List[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -217,7 +228,7 @@ def get_func_args(func, stripself=False): return args -def get_spec(func): +def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -246,7 +257,7 @@ def get_spec(func): else: raise TypeError(f"{type(func)} is not callable") - defaults = spec.defaults or [] + defaults: Tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -254,7 +265,9 @@ def get_spec(func): return args, kwargs -def equal_attributes(obj1, obj2, attributes): +def equal_attributes( + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] +) -> bool: """Compare two objects attributes""" # not attributes given return False by default if not attributes: @@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable: ... -def without_none_values(iterable): +def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - try: + if isinstance(iterable, collections.abc.Mapping): return {k: v for k, v in iterable.items() if v is not None} - except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + else: + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] -def global_object_name(obj): +def global_object_name(obj: Any) -> str: """ Return full name of a global object. @@ -307,14 +321,14 @@ def global_object_name(obj): if hasattr(sys, "pypy_version_info"): - def garbage_collect(): + def garbage_collect() -> None: # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() else: - def garbage_collect(): + def garbage_collect() -> None: gc.collect() @@ -329,10 +343,10 @@ class MutableChain(Iterable): def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self): + def __iter__(self) -> Iterator: return self - def __next__(self): + def __next__(self) -> Any: return next(self.data) @@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable): def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self): + def __aiter__(self) -> AsyncIterator: return self - async def __anext__(self): + async def __anext__(self) -> Any: return await self.data.__anext__() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f1b9239e6..ad3d1d8bc 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,7 +1,8 @@ import asyncio import sys +from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence +from typing import Any, Callable, Dict, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -57,7 +58,7 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def set_asyncio_event_loop_policy(): +def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. This should only be used to install the reactor. @@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy(): _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy(): +def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: warn( "Call to deprecated function " "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" @@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy(): return _get_asyncio_event_loop_policy() -def _get_asyncio_event_loop_policy(): +def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy(): return policy -def install_reactor(reactor_path, event_loop_path=None): +def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None): installer() -def _get_asyncio_event_loop(): +def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path): +def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) + event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: @@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path): return event_loop -def verify_installed_reactor(reactor_path): +def verify_installed_reactor(reactor_path: str) -> None: """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import path.""" @@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) -def verify_installed_asyncio_event_loop(loop_path): +def verify_installed_asyncio_event_loop(loop_path: str) -> None: from twisted.internet import reactor loop_class = load_object(loop_path) @@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path): ) -def is_asyncio_reactor_installed(): +def is_asyncio_reactor_installed() -> bool: from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py deleted file mode 100644 index 15705db83..000000000 --- a/scrapy/utils/reqser.py +++ /dev/null @@ -1,27 +0,0 @@ -import warnings -from typing import Optional - -import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.request import request_from_dict as _from_dict - -warnings.warn( - ( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead" - ), - category=ScrapyDeprecationWarning, - stacklevel=2, -) - - -def request_to_dict( - request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None -) -> dict: - return request.to_dict(spider=spider) - - -def request_from_dict( - d: dict, spider: Optional["scrapy.Spider"] = None -) -> "scrapy.Request": - return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6d8be991d..24fcbd85e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -6,7 +6,19 @@ scrapy.http.Request objects import hashlib import json import warnings -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + Union, +) from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -19,11 +31,16 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy.crawler import Crawler + _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() -def _serialize_headers(headers, request): +def _serialize_headers( + headers: Iterable[bytes], request: Request +) -> Generator[bytes, Any, None]: for header in headers: if header in request.headers: yield header @@ -139,7 +156,7 @@ def request_fingerprint( return cache[cache_key] -def _request_fingerprint_as_bytes(*args, **kwargs): +def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: with warnings.catch_warnings(): warnings.simplefilter("ignore") return bytes.fromhex(request_fingerprint(*args, **kwargs)) @@ -214,6 +231,11 @@ def fingerprint( return cache[cache_key] +class RequestFingerprinterProtocol(Protocol): + def fingerprint(self, request: Request) -> bytes: + ... + + class RequestFingerprinter: """Default fingerprinter. @@ -231,7 +253,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler): return cls(crawler) - def __init__(self, crawler=None): + def __init__(self, crawler: Optional["Crawler"] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -265,7 +287,7 @@ class RequestFingerprinter: f"and '2.7'." ) - def fingerprint(self, request: Request): + def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) @@ -311,7 +333,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls = load_object(d["_class"]) if "_class" in d else Request + request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) @@ -320,7 +342,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: return request_cls(**kwargs) -def _get_method(obj, name): +def _get_method(obj: Any, name: Any) -> Any: """Helper function for request_from_dict""" name = str(name) try: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 794678c48..77d54aff9 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -14,7 +14,6 @@ from w3lib import html import scrapy from scrapy.http.response import Response -from scrapy.utils.decorators import deprecated from scrapy.utils.python import to_bytes, to_unicode _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() @@ -42,8 +41,7 @@ def get_meta_refresh( """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 - _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] + _metaref_cache[response] = html.get_meta_refresh( text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] @@ -56,25 +54,6 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" -@deprecated -def response_httprepr(response: Response) -> bytes: - """Return raw HTTP representation (as bytes) of the given response. This - is provided only for reference, since it's not the exact stream of bytes - that was received (that's not exposed by Twisted). - """ - values = [ - b"HTTP/1.1 ", - to_bytes(str(response.status)), - b" ", - to_bytes(http.RESPONSES.get(response.status, b"")), - b"\r\n", - ] - if response.headers: - values.extend([response.headers.to_string(), b"\r\n"]) - values.extend([b"\r\n", response.body]) - return b"".join(values) - - def open_in_browser( response: Union[ "scrapy.http.response.html.HtmlResponse", diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 414658944..3b4f67f00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,6 +1,7 @@ import datetime import decimal import json +from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer @@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" - def default(self, o): + def default(self, o: Any) -> Any: if isinstance(o, set): return list(o) if isinstance(o, datetime.datetime): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 9e7ddd827..21a12a19e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) def send_catch_log( - signal=Any, sender=Anonymous, *arguments, **named + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -65,13 +68,18 @@ def send_catch_log( return responses -def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log_deferred( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny +) -> Deferred: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. """ - def logerror(failure, recv): + def logerror(failure: Failure, recv: Any) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): return d -def disconnect_all(signal=Any, sender=Any): +def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running tests """ diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2622c2775..3d2ecc9a7 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ - +from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin import lxml.etree @@ -14,7 +14,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext): + def __init__(self, xmltext: str): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -22,9 +22,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self): + def __iter__(self) -> Iterator[Dict[str, Any]]: for elem in self._root.getchildren(): - d = {} + d: Dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag @@ -39,11 +39,13 @@ class Sitemap: yield d -def sitemap_urls_from_robots(robots_text, base_url=None): +def sitemap_urls_from_robots( + robots_text: str, base_url: Optional[str] = None +) -> Generator[str, Any, None]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith("sitemap:"): url = line.split(":", 1)[1].strip() - yield urljoin(base_url, url) + yield urljoin(base_url or "", url) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 86449eeb2..704df8657 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,14 +1,54 @@ +from __future__ import annotations + import inspect import logging +from types import CoroutineType, ModuleType +from typing import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Generator, + Iterable, + Literal, + Optional, + Type, + TypeVar, + Union, + overload, +) +from twisted.internet.defer import Deferred + +from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) +_T = TypeVar("_T") -def iterate_spider_output(result): + +# https://stackoverflow.com/questions/60222982 +@overload +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] + ... + + +@overload +def iterate_spider_output(result: CoroutineType) -> Deferred: + ... + + +@overload +def iterate_spider_output(result: _T) -> Iterable: + ... + + +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -18,7 +58,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module): +def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -36,9 +76,46 @@ def iter_spider_classes(module): yield obj +@overload def spidercls_for_request( - spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False -): + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Type[Spider], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Type[Spider]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Literal[None], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + *, + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Optional[Type[Spider]] = None, + log_none: bool = False, + log_multiple: bool = False, +) -> Optional[Type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 03ae4ba9e..e74769c65 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ -from typing import Any, Optional, cast +from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil +import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name @@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]: def get_openssl_version() -> str: - # https://github.com/python/typeshed/issues/10024 - system_openssl_bytes = cast( - bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - ) + system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1499aeb3d..6b22f3bfa 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -4,10 +4,10 @@ import re import string from os import PathLike from pathlib import Path -from typing import Union +from typing import Any, Union -def render_templatefile(path: Union[str, PathLike], **kwargs): +def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") @@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]") -def string_camelcase(string): +def string_camelcase(string: str) -> str: """Convert a word to its CamelCase version and remove invalid chars >>> string_camelcase('lost-pound') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 58576903a..709e0b00d 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,24 +7,30 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from unittest import mock +from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from unittest import TestCase, mock +from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available -def assert_gcs_environ(): +def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") -def skip_if_no_boto(): +def skip_if_no_boto() -> None: if not is_botocore_available(): raise SkipTest("missing botocore library") -def get_gcs_content_and_delete(bucket, path): +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> Tuple[bytes, List[Dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path): def get_ftp_content_and_delete( - path, host, port, username, password, use_active_mode=False -): + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: from ftplib import FTP ftp = FTP() @@ -46,33 +57,42 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data = [] + ftp_data: List[bytes] = [] - def buffer_data(data): + def buffer_data(data: bytes) -> None: ftp_data.append(data) ftp.retrbinary(f"RETR {path}", buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) - return "".join(ftp_data) + return b"".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): +class TestSpider(Spider): + name = "test" + + +def get_crawler( + spidercls: Optional[Type[Spider]] = None, + settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, +) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ from scrapy.crawler import CrawlerRunner - from scrapy.spiders import Spider # Set by default settings that prevent deprecation warnings. - settings = {} + settings: Dict[str, Any] = {} if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - return runner.create_crawler(spidercls or Spider) + crawler = runner.create_crawler(spidercls or TestSpider) + crawler._apply_settings() + return crawler def get_pythonpath() -> str: @@ -82,7 +102,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv(): +def get_testenv() -> Dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -91,21 +111,23 @@ def get_testenv(): return env -def assert_samelines(testcase, text1, text2, msg=None): +def assert_samelines( + testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None +) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value): - q = asyncio.Queue() +def get_from_asyncio_queue(value: Any) -> Coroutine: + q: asyncio.Queue = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter -def mock_google_cloud_storage(): +def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ @@ -122,7 +144,7 @@ def mock_google_cloud_storage(): return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url): +def get_web_client_agent_req(url: str) -> Deferred: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f9bdef37..3bdffcaa7 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,49 +1,63 @@ +from __future__ import annotations + import os import sys +from typing import Iterable, List, Optional, Tuple, cast -from twisted.internet import defer, protocol +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol +from twisted.python.failure import Failure class ProcessTest: - command = None + command: Optional[str] = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir - def execute(self, args, check_code=True, settings=None): + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: Optional[str] = None, + ) -> Deferred: from twisted.internet import reactor env = os.environ.copy() if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command cmd = self.prefix + [self.command] + list(args) pp = TestProcessProtocol() - pp.deferred.addBoth(self._process_finished, cmd, check_code) + pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred - def _process_finished(self, pp, cmd, check_code): + def _process_finished( + self, pp: TestProcessProtocol, cmd: List[str], check_code: bool + ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err}" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" raise RuntimeError(msg) - return pp.exitcode, pp.out, pp.err + return cast(int, pp.exitcode), pp.out, pp.err -class TestProcessProtocol(protocol.ProcessProtocol): - def __init__(self): - self.deferred = defer.Deferred() - self.out = b"" - self.err = b"" - self.exitcode = None +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: Optional[int] = None - def outReceived(self, data): + def outReceived(self, data: bytes) -> None: self.out += data - def errReceived(self, data): + def errReceived(self, data: bytes) -> None: self.err += data - def processEnded(self, status): - self.exitcode = status.value.exitCode + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode self.deferred.callback(self) diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 01b980c93..9ff9a273f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -12,9 +12,14 @@ alias to object in that case). from collections import defaultdict from operator import itemgetter from time import time -from typing import DefaultDict +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable from weakref import WeakKeyDictionary +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) @@ -24,13 +29,14 @@ class object_ref: __slots__ = () - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> "Self": obj = object.__new__(cls) live_refs[cls][obj] = time() return obj -def format_live_refs(ignore=NoneType): +# using Any as it's hard to type type(None) +def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() @@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType): return s -def print_live_refs(*a, **kw): +def print_live_refs(*a: Any, **kw: Any) -> None: """Print tracked objects""" print(format_live_refs(*a, **kw)) -def get_oldest(class_name): +def get_oldest(class_name: str) -> Any: """Get the oldest object for a specific class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: @@ -58,8 +64,9 @@ def get_oldest(class_name): return min(wdict.items(), key=itemgetter(1))[0] -def iter_all(class_name): +def iter_all(class_name: str) -> Iterable[Any]: """Iterate over all objects of the same class by its class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() + return [] diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 833aa3e20..22b4197f9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import re +from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from scrapy import Spider -def url_is_from_any_domain(url, domains): + +UrlT = Union[str, bytes, ParseResult] + + +def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc.lower() if not host: @@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains): return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url, spider): +def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) ) -def url_has_any_extension(url, extensions): +def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url, encoding=None): +def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ if isinstance(url, ParseResult): return url - return urlparse(to_unicode(url, encoding)) + return cast(ParseResult, urlparse(to_unicode(url, encoding))) -def escape_ajax(url): +def escape_ajax(url: str) -> str: """ Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started @@ -76,7 +83,7 @@ def escape_ajax(url): return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) -def add_http_if_no_scheme(url): +def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) if not match: @@ -87,7 +94,7 @@ def add_http_if_no_scheme(url): return url -def _is_posix_path(string): +def _is_posix_path(string: str) -> bool: return bool( re.match( r""" @@ -109,7 +116,7 @@ def _is_posix_path(string): ) -def _is_windows_path(string): +def _is_windows_path(string: str) -> bool: return bool( re.match( r""" @@ -125,11 +132,11 @@ def _is_windows_path(string): ) -def _is_filesystem_path(string): +def _is_filesystem_path(string: str) -> bool: return _is_posix_path(string) or _is_windows_path(string) -def guess_scheme(url): +def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): @@ -138,12 +145,12 @@ def guess_scheme(url): def strip_url( - url, - strip_credentials=True, - strip_default_port=True, - origin_only=False, - strip_fragment=True, -): + url: str, + strip_credentials: bool = True, + strip_default_port: bool = True, + origin_only: bool = False, + strip_fragment: bool = True, +) -> str: """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index b0737d3d5..9b637bdb0 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,5 +1,6 @@ import platform import sys +from typing import List, Tuple import cryptography import cssselect @@ -12,7 +13,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions(): +def scrapy_components_versions() -> List[Tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index e8affa943..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,113 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) - -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -Managing add-ons -================ - -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. - -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: - - [addons] - httpcache = - -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: - - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py - - -Writing add-ons -=============== - -Add-ons are Python modules that implement the following callbacks. - -addon_configure ---------------- - -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. - -Examples: - -.. code-block:: python - - def addon_configure(settings): - settings.overrides["DOWNLOADER_MIDDLEWARES"].update( - { - "scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware": 900, - } - ) - -.. code-block:: python - - def addon_configure(settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - - -crawler_ready -------------- - -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. - -Examples: - -.. code-block:: python - - def crawler_ready(crawler): - if "some.other.addon" not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") diff --git a/setup.cfg b/setup.cfg index db79c5821..151e784c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,85 +8,17 @@ universal=1 ignore_missing_imports = true # Interface classes are hard to support + [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-scrapy.interfaces] +ignore_errors = True + [mypy-twisted.internet.reactor] follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy.downloadermiddlewares.httpproxy] -ignore_errors = True - -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-scrapy.pipelines.images] -ignore_errors = True - [mypy-scrapy.settings.default_settings] ignore_errors = True - -[mypy-tests.mocks.dummydbm] -ignore_errors = True - -[mypy-tests.test_command_fetch] -ignore_errors = True - -[mypy-tests.test_command_parse] -ignore_errors = True - -[mypy-tests.test_command_shell] -ignore_errors = True - -[mypy-tests.test_command_version] -ignore_errors = True - -[mypy-tests.test_contracts] -ignore_errors = True - -[mypy-tests.test_downloader_handlers] -ignore_errors = True - -[mypy-tests.test_exporters] -ignore_errors = True - -[mypy-tests.test_http_request] -ignore_errors = True - -[mypy-tests.test_linkextractors] -ignore_errors = True - -[mypy-tests.test_loader] -ignore_errors = True - -[mypy-tests.test_loader_deprecated] -ignore_errors = True - -[mypy-tests.test_pipeline_crawl] -ignore_errors = True - -[mypy-tests.test_pipeline_files] -ignore_errors = True - -[mypy-tests.test_pipeline_images] -ignore_errors = True - -[mypy-tests.test_request_cb_kwargs] -ignore_errors = True - -[mypy-tests.test_scheduler] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_httperror] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_referer] -ignore_errors = True - -[mypy-tests.test_utils_serialize] -ignore_errors = True - -[mypy-tests.test_utils_url] -ignore_errors = True diff --git a/setup.py b/setup.py index c6bcf2439..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,13 @@ from pathlib import Path -from pkg_resources import parse_version -from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() -def has_environment_marker_platform_impl_support(): - """Code extracted from 'pytest/setup.py' - https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 - - The first known release to support environment marker with range operators - it is 18.5, see: - https://setuptools.readthedocs.io/en/latest/history.html#id235 - """ - return parse_version(setuptools_version) >= parse_version("18.5") - - install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.4.6", + "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", @@ -34,21 +21,12 @@ install_requires = [ "setuptools", "packaging", "tldextract", - "lxml>=4.3.0", + "lxml>=4.4.1", ] -extras_require = {} -cpython_dependencies = [ - "PyDispatcher>=2.0.5", -] -if has_environment_marker_platform_impl_support(): - extras_require[ - ':platform_python_implementation == "CPython"' - ] = cpython_dependencies - extras_require[':platform_python_implementation == "PyPy"'] = [ - "PyPyDispatcher>=2.1.0", - ] -else: - install_requires.extend(cpython_dependencies) +extras_require = { + ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], + ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], +} setup( @@ -80,18 +58,18 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py new file mode 100644 index 000000000..a46a8806b --- /dev/null +++ b/tests/CrawlerProcess/args_settings.py @@ -0,0 +1,24 @@ +from typing import Any + +import scrapy +from scrapy.crawler import Crawler, CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + def start_requests(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 34ef00143..9dc8ce46b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -2,6 +2,7 @@ import asyncio import sys from twisted.internet import asyncioreactor +from twisted.python import log if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) @@ -24,5 +25,6 @@ process = CrawlerProcess( "ASYNCIO_EVENT_LOOP": "uvloop.Loop", } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 744b4ecb5..eee808c32 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -17,5 +18,6 @@ process = CrawlerProcess( } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index a8f707841..38ca4c4f1 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,5 +1,6 @@ from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -26,5 +27,6 @@ process = CrawlerProcess( } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py new file mode 100644 index 000000000..45479ea4f --- /dev/null +++ b/tests/CrawlerProcess/sleeping.py @@ -0,0 +1,24 @@ +from twisted.internet.defer import Deferred + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import maybe_deferred_to_future + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(int(self.sleep), d.callback, None) + await maybe_deferred_to_future(d) + + +process = CrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 19cc08be6..d71014b34 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -1,3 +1,5 @@ +from twisted.python import log + import scrapy from scrapy.crawler import CrawlerProcess @@ -17,6 +19,8 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() -process.crawl(SelectReactorSpider) -process.crawl(AsyncioReactorSpider) +d1 = process.crawl(SelectReactorSpider) +d1.addErrback(log.err) +d2 = process.crawl(AsyncioReactorSpider) +d2.addErrback(log.err) process.start() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 5cc65a903..9b73ca4f0 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from cryptography.hazmat.backends import default_backend @@ -50,8 +50,8 @@ def generate_keys(): .issuer_name(issuer) .public_key(key.public_key()) .serial_number(random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .not_valid_before(datetime.now(tz=timezone.utc)) + .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10)) .add_extension( SubjectAlternativeName([DNSName("localhost")]), critical=False, diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e77f53666..2869ff8f7 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,5 +1,6 @@ """DBM-like dummy module""" import collections +from typing import Any, DefaultDict class DummyDB(dict): @@ -12,7 +13,7 @@ class DummyDB(dict): error = KeyError -_DATABASES = collections.defaultdict(DummyDB) +_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/mockserver.py b/tests/mockserver.py index eb4f8334e..647b0682e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,11 +1,13 @@ import argparse import json +import os import random import sys from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import Dict from urllib.parse import urlencode from OpenSSL import SSL @@ -20,7 +22,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None): return default +def get_mockserver_env() -> Dict[str, str]: + """Return a OS environment dict suitable to run mockserver processes.""" + + tests_path = Path(__file__).parent.parent + pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") + env = os.environ.copy() + env["PYTHONPATH"] = pythonpath + return env + + # most of the following resources are copied from twisted.web.test.test_webclient class ForeverTakingResource(resource.Resource): """ @@ -264,7 +275,7 @@ class MockServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) http_address = self.proc.stdout.readline().strip().decode("ascii") https_address = self.proc.stdout.readline().strip().decode("ascii") @@ -308,7 +319,7 @@ class MockDNSServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) self.host = "127.0.0.1" self.port = int( @@ -331,7 +342,7 @@ class MockFTPServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)], stderr=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) for line in self.proc.stderr: if b"starting FTP server" in line: diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..5b75674f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ # Tests requirements attrs -pyftpdlib +pexpect >= 4.8.0 +pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 pytest-xdist @@ -8,9 +9,10 @@ sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-6054334 testfixtures uvloop; platform_system != "Windows" -# optional for shell wrapper tests -bpython -brotli # optional for HTTP compress downloader middleware tests +bpython # optional for shell wrapper tests +brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests +# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 +brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/spiders.py b/tests/spiders.py index 6ff48f471..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -77,6 +77,22 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class SlowSpider(DelaySpider): + name = "slow" + + def start_requests(self): + # 1st response is fast + url = self.mockserver.url("/delay?n=0&b=0") + yield Request(url, callback=self.parse, errback=self.errback) + + # 2nd response is slow + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") + yield Request(url, callback=self.parse, errback=self.errback) + + def parse(self, response): + yield Item() + + class SimpleSpider(MetaSpider): name = "simple" diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..0f4f2e5b8 --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,202 @@ +import itertools +from typing import Any, Dict +from unittest.mock import patch + +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest + +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured +from scrapy.settings import BaseSettings, Settings +from scrapy.utils.test import get_crawler + + +class SimpleAddon: + def update_settings(self, settings): + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig + + +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + +class AddonTest(unittest.TestCase): + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = get_addon_cls(addon_config)() + testaddon.update_settings(settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def test_load_settings(self): + settings_dict = { + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + crawler._apply_settings() + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + + def test_logging_message(self): + class LoggedAddon: + def update_settings(self, settings): + pass + + with patch("scrapy.addons.logger") as logger_mock: + with patch("scrapy.addons.create_instance") as create_instance_mock: + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + create_instance_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) + + @inlineCallbacks + def test_enable_addon_in_spider(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + addon_config = {"KEY": "addon"} + addon_cls = get_addon_cls(addon_config) + spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider") + return spider + + settings = Settings() + settings.set("KEY", "default", priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(MySpider) + self.assertEqual(crawler.settings.get("KEY"), "default") + yield crawler.crawl() + self.assertEqual(crawler.settings.get("KEY"), "addon") diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9b39187d5..38ede70e4 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider class TestCloseSpider(TestCase): @@ -54,3 +54,13 @@ class TestCloseSpider(TestCase): self.assertEqual(reason, "closespider_timeout") total_seconds = crawler.stats.get_value("elapsed_time_seconds") self.assertTrue(total_seconds >= close_on) + + @defer.inlineCallbacks + def test_closespider_timeout_no_item(self): + timeout = 1 + crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_timeout_no_item") + total_seconds = crawler.stats.get_value("elapsed_time_seconds") + self.assertTrue(total_seconds >= timeout) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 15833cd19..25ded143c 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,4 +1,5 @@ import json +import os import pstats import shutil import sys @@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv class CmdlineTest(unittest.TestCase): def setUp(self): self.env = get_testenv() + tests_path = Path(__file__).parent.parent + self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6589381f3..7918d94b2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,11 +1,16 @@ +import os +import sys +from io import BytesIO from pathlib import Path +from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.mockserver import MockServer class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -133,3 +138,27 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + + +class InteractiveShellTest(unittest.TestCase): + def test_fetch(self): + args = ( + sys.executable, + "-m", + "scrapy.cmdline", + "shell", + ) + env = os.environ.copy() + env["SCRAPY_PYTHON_SHELL"] = "python" + logfile = BytesIO() + p = PopenSpawn(args, env=env, timeout=5) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + with MockServer() as mockserver: + p.sendline(f"fetch('{mockserver.url('/')}')") + p.sendline("type(response)") + p.expect_exact("HtmlResponse") + p.sendeof() + p.wait() + logfile.seek(0) + self.assertNotIn("Traceback", logfile.read().decode()) diff --git a/tests/test_commands.py b/tests/test_commands.py index 014f50e92..36f800850 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -802,17 +800,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, @@ -919,6 +907,86 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", + log, + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", + log, + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", + log, + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", + log, + ) + + def test_args_change_settings(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + def start_requests(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return [] +""" + args = ["-a", "foo=42"] + log = self.get_log(spider_code, args=args) + self.assertIn("Spider closed (finished)", log) + self.assertIn("The value of FOO is 42", log) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 813927fc5..1459e0b5f 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -63,6 +63,13 @@ class TestSpider(Spider): """ return Request("http://scrapy.org", callback=self.returns_item) + async def returns_request_async(self, response): + """async method which returns request + @url http://scrapy.org + @returns requests 1 + """ + return Request("http://scrapy.org", callback=self.returns_item) + def returns_item(self, response): """method which returns item @url http://scrapy.org @@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response) self.should_fail() + def test_returns_async(self): + spider = TestSpider() + response = ResponseMock() + + request = self.conman.from_method(spider.returns_request_async, self.results) + request.callback(response) + self.should_error() + def test_scrapes(self): spider = TestSpider() response = ResponseMock() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ca9084294..96d43b2b9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -123,7 +123,9 @@ class CrawlTestCase(TestCase): self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + # server hangs after receiving response headers + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) @@ -201,6 +203,7 @@ class CrawlTestCase(TestCase): ) self.assertEqual(crawler.spider.visited, 6) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 706bfbaa9..92bd5f38f 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,17 +1,21 @@ import logging +import os import platform +import signal import subprocess import sys import warnings from pathlib import Path +from typing import List -from pkg_resources import parse_version +import pytest +from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises -from twisted import version as twisted_version from twisted.internet import defer -from twisted.python.versions import Version from twisted.trial import unittest from w3lib import __version__ as w3lib_version +from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner @@ -21,10 +25,9 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler, get_testenv -from tests.mockserver import MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer, get_mockserver_env class BaseCrawlerTest(unittest.TestCase): @@ -43,9 +46,8 @@ class CrawlerTestCase(BaseCrawlerTest): settings = Settings() settings.setdict(project_settings, priority="project") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(CustomSettingsSpider, settings) + crawler = Crawler(CustomSettingsSpider, settings) + crawler._apply_settings() self.assertEqual(crawler.settings.get("TEST1"), "spider") self.assertEqual(crawler.settings.get("TEST2"), "spider") @@ -69,6 +71,16 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) + @defer.inlineCallbacks + def test_crawler_crawl_twice_deprecated(self): + crawler = Crawler(NoRequestsSpider) + yield crawler.crawl() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Running Crawler.crawl\(\) more than once is deprecated", + ): + yield crawler.crawl() + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -168,11 +180,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - with warnings.catch_warnings(record=True) as w: - self.assertRaises(AttributeError, CrawlerRunner, settings) - self.assertEqual(len(w), 1) - self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) - self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) + self.assertRaises(MultipleInvalid, CrawlerRunner, settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) @@ -183,16 +191,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - runner = CrawlerRunner(Settings()) - spiders = runner.spiders - self.assertEqual(len(w), 1) - self.assertIn("CrawlerRunner.spiders", str(w[0].message)) - self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) - sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"]) - self.assertIsInstance(spiders, sl_cls) - class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): @@ -229,14 +227,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -249,7 +247,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: self.fail("Exception should be raised from spider") - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -264,7 +262,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): @@ -289,12 +287,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path + cwd = os.getcwd() - def run_script(self, script_name: str, *script_args): + def get_script_args(self, script_name: str, *script_args: str) -> List[str]: script_path = self.script_dir / script_name - args = [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path)] + list(script_args) + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) p = subprocess.Popen( - args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + args, + env=get_mockserver_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() return stderr.decode("utf-8") @@ -461,17 +466,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -480,17 +475,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -500,17 +485,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -519,17 +494,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) @@ -550,6 +515,41 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + def test_args_change_settings(self): + log = self.run_script("args_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("The value of FOO is 42", log) + + def test_shutdown_graceful(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py", "-a", "sleep=3") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() + + @defer.inlineCallbacks + def test_shutdown_forced(self): + from twisted.internet import reactor + + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py", "-a", "sleep=10") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + d = defer.Deferred() + reactor.callLater(0.1, d.callback, None) + yield d + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fd4176e2f..924ece6f9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -127,9 +127,9 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f"file://{self.mktemp()}") + request = Request(path_to_file_uri(self.mktemp())) d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, IOError) + return self.assertFailure(d, OSError) class ContentLengthHeaderResource(resource.Resource): @@ -726,32 +726,6 @@ class Http11MockServerTestCase(unittest.TestCase): reason = crawler.spider.meta["close_reason"] self.assertTrue(reason, "finished") - @defer.inlineCallbacks - def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request( - self.mockserver.url("/payload"), - method="POST", - body=body, - meta={"download_maxsize": 50}, - ) - yield crawler.crawl(seed=request) - failure = crawler.spider.meta["failure"] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload fails on PY3") - request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate") - request = request.replace(url=self.mockserver.url("/xpayload")) - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) - reason = crawler.spider.meta["close_reason"] - self.assertTrue(reason, "finished") - class UriResource(resource.Resource): """Return the full uri that was requested""" diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 060cfe08b..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase): In particular when some website returns a 30x response with header 'Content-Encoding: gzip' giving as result the error below: - exceptions.IOError: Not a gzipped file + BadGzipFile: Not a gzipped file (...) """ req = Request("http://example.com") @@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(IOError, self._download, request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py deleted file mode 100644 index 95739414e..000000000 --- a/tests/test_downloadermiddleware_decompression.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase, main - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.http import Response, XmlResponse -from scrapy.spiders import Spider -from scrapy.utils.test import assert_samelines -from tests import get_testdata - - -def _test_data(formats): - uncompressed_body = get_testdata("compressed", "feed-sample1.xml") - test_responses = {} - for format in formats: - body = get_testdata("compressed", "feed-sample1." + format) - test_responses[format] = Response("http://foo.com/bar", body=body) - return uncompressed_body, test_responses - - -class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] - uncompressed_body, test_responses = _test_data(test_formats) - - def setUp(self): - self.mw = DecompressionMiddleware() - self.spider = Spider("foo") - - def test_known_compression_formats(self): - for fmt in self.test_formats: - rsp = self.test_responses[fmt] - new = self.mw.process_response(None, rsp, self.spider) - error_msg = f"Failed {fmt}, response type {type(new).__name__}" - assert isinstance(new, XmlResponse), error_msg - assert_samelines(self, new.body, self.uncompressed_body, fmt) - - def test_plain_response(self): - rsp = Response(url="http://test.com", body=self.uncompressed_body) - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert_samelines(self, new.body, rsp.body) - - def test_empty_response(self): - rsp = Response(url="http://test.com", body=b"") - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert not rsp.body - assert not new.body - - def tearDown(self): - del self.mw - - -if __name__ == "__main__": - main() diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 6b79234d0..fc110e6cc 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,10 +1,8 @@ import unittest -import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider @@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = TestSpiderLegacy("foo") def test_auth(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): + with self.assertRaises(AttributeError): mw = HttpAuthMiddleware() mw.spider_opened(self.spider) - # initial request, sets the domain and sends the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to the same domain, should send the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to a different domain, shouldn't send the header - req = Request("http://example-noauth.com/") - assert mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) - - def test_auth_already_set(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f9e9b4661..0c4fff629 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path from unittest import SkipTest, TestCase -from warnings import catch_warnings from w3lib.encoding import resolve_encoding @@ -10,7 +9,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.settings import Settings @@ -241,10 +240,10 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) self.assertIn('Content-Encoding', newresponse.headers) self.assertEqual(b'gzip', newresponse.headers['Content-Encoding']) self.assertIn(b'decoded', newresponse.flags) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_force_recalculate_encoding(self): headers = { @@ -271,8 +270,8 @@ class HttpCompressionTest(TestCase): self.assertIn('Content-Encoding', newresponse.headers) self.assertEqual(b'gzip', newresponse.headers['Content-Encoding']) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) self.assertIn(b'decoded', newresponse.flags) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): headers = { @@ -297,8 +296,8 @@ class HttpCompressionTest(TestCase): self.assertIn('Content-Encoding', newresponse.headers) self.assertEqual(b'identity', newresponse.headers['Content-Encoding']) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) self.assertIn(b'decode', newresponse.flags) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 63bd61848..661175840 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,6 @@ import logging import unittest +import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -110,19 +112,63 @@ class RetryTest(unittest.TestCase): == 2 ) - def _test_retry_exception(self, req, exception): + def test_exception_to_retry_added(self): + exc = ValueError + settings_dict = { + "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + } + crawler = get_crawler(Spider, settings_dict=settings_dict) + mw = RetryMiddleware.from_crawler(crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + self._test_retry_exception(req, exc("foo"), mw) + + def test_exception_to_retry_custom_middleware(self): + exc = ValueError + + with warnings.catch_warnings(record=True) as warns: + + class MyRetryMiddleware(RetryMiddleware): + EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) + + self.assertEqual(len(warns), 1) + + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + + def test_exception_to_retry_custom_middleware_self(self): + class MyRetryMiddleware(RetryMiddleware): + def process_exception(self, request, exception, spider): + if isinstance(exception, self.EXCEPTIONS_TO_RETRY): + return self._retry(request, exception, spider) + + exc = OSError + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + with warnings.catch_warnings(record=True) as warns: + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + self.assertEqual(len(warns), 1) + + def _test_retry_exception(self, req, exception, mw=None): + if mw is None: + mw = self.mw + # first retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 1) # second retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 2) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 39dfe9ab5..5b7181848 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,12 +1,8 @@ -import warnings -from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spiders import Spider -from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -40,25 +36,6 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual("downloader/response_count", 1) - def test_response_len(self): - body = (b"", b"not_empty") # empty/notempty body - headers = ( - {}, - {"lang": "en"}, - {"lang": "en", "User-Agent": "scrapy"}, - ) # 0 headers, 1h and 2h - test_responses = [ # form test responses with all combinations of body/headers - Response(url="scrapytest.org", status=200, body=r[0], headers=r[1]) - for r in product(body, headers) - ] - for test_response in test_responses: - self.crawler.stats.set_value("downloader/response_bytes", 0) - self.mw.process_response(self.req, test_response, self.spider) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - resp_size = len(response_httprepr(test_response)) - self.assertStatsEqual("downloader/response_bytes", resp_size) - def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual("downloader/exception_count", 1) diff --git a/tests/test_engine.py b/tests/test_engine.py index 410eba921..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -20,7 +20,6 @@ from threading import Timer from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -29,7 +28,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor @@ -436,90 +435,6 @@ class EngineTest(unittest.TestCase): finally: yield e.stop() - @defer.inlineCallbacks - def test_close_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_crawl_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.crawl(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_download_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.download(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_schedule(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.schedule(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_has_capacity(self): - with pytest.warns( - ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated" - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - self.assertTrue(e.has_capacity()) - spider = TestSpider() - yield e.open_spider(spider, []) - self.assertFalse(e.has_capacity()) - e.start() - yield e.close() - self.assertTrue(e.has_capacity()) - def test_short_timeout(self): args = ( sys.executable, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1491e788e..c11913365 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,12 +7,11 @@ import tempfile import unittest from datetime import datetime from io import BytesIO -from warnings import catch_warnings, filterwarnings +from typing import Any import lxml.etree from itemadapter import ItemAdapter -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, CsvItemExporter, @@ -55,8 +54,8 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class = TestItem - custom_field_item_class = CustomFieldItem + item_class: type = TestItem + custom_field_item_class: type = CustomFieldItem def setUp(self): self.i = self.item_class(name="John\xa3", age="22") @@ -90,6 +89,10 @@ class BaseItemExporterTest(unittest.TestCase): if self.ie.__class__ is not BaseItemExporter: raise self.ie.finish_exporting() + # Delete the item exporter object, so that if it causes the output + # file handle to be closed, which should not be the case, follow-up + # interactions with the output file handle will surface the issue. + del self.ie self._check_output() def test_export_item(self): @@ -139,7 +142,7 @@ class BaseItemExporterDataclassTest(BaseItemExporterTest): class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(binary=False, **kwargs) + return PythonItemExporter(**kwargs) def test_invalid_option(self): with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): @@ -194,14 +197,6 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported["age"][0]), dict) self.assertEqual(type(exported["age"][0]["age"][0]), dict) - def test_export_binary(self): - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - exporter = PythonItemExporter(binary=True) - value = self.item_class(name="John\xa3", age="22") - expected = {b"name": b"John\xc2\xa3", b"age": b"22"} - self.assertEqual(expected, exporter.export_item(value)) - def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() @@ -243,6 +238,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. f.seek(0) self.assertEqual(self.item_class(**pickle.load(f)), i1) self.assertEqual(self.item_class(**pickle.load(f)), i2) @@ -254,6 +250,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertEqual(pickle.loads(fp.getvalue()), item) @@ -279,6 +276,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. fp.seek(0) self.assertEqual(marshal.load(fp), item) @@ -314,6 +312,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(fp.getvalue(), expected) def test_header_export_all(self): @@ -345,6 +344,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual( output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n" ) @@ -429,6 +429,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): @@ -517,7 +518,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = { + _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, } @@ -536,6 +537,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) @@ -550,6 +552,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) self.assertEqual(exported, item) @@ -575,6 +578,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual( exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] @@ -586,6 +590,20 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def test_two_dict_items(self): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph\xa3", age="22") + i2 = TestItem( + name="Maria", age=1j + ) # Invalid datetimes didn't consistently fail between Python versions + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(TypeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + self.assertEqual(exported, [dict(i1), dict(i3)]) + def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") i2 = self.item_class(name="Maria", age=i1) @@ -593,6 +611,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = { "name": "Jesus", @@ -607,6 +626,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} self.assertEqual(exported, [expected]) @@ -616,18 +636,37 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) self.assertEqual(exported, [item]) +class JsonItemExporterToBytesTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + kwargs["encoding"] = "latin" + return JsonItemExporter(self.output, **kwargs) + + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph", age="22") + i2 = TestItem(name="\u263a", age="11") + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) + self.assertEqual(exported, [dict(i1), dict(i3)]) + + class JsonItemExporterDataclassTest(JsonItemExporterTest): item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class = TestItem + item_class: type = TestItem def setUp(self): if self.item_class is None: diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py new file mode 100644 index 000000000..502ada6be --- /dev/null +++ b/tests/test_extension_periodic_log.py @@ -0,0 +1,195 @@ +import datetime +import typing +import unittest + +from scrapy.crawler import Crawler +from scrapy.extensions.periodic_log import PeriodicLog + +from .spiders import MetaSpider + +stats_dump_1 = { + "log_count/INFO": 10, + "log_count/WARNING": 1, + "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170), + "scheduler/enqueued/memory": 190, + "scheduler/enqueued": 190, + "scheduler/dequeued/memory": 166, + "scheduler/dequeued": 166, + "downloader/request_count": 166, + "downloader/request_method_count/GET": 166, + "downloader/request_bytes": 56803, + "downloader/response_count": 150, + "downloader/response_status_count/200": 150, + "downloader/response_bytes": 595698, + "httpcompression/response_bytes": 3186068, + "httpcompression/response_count": 150, + "response_received_count": 150, + "request_depth_max": 9, + "dupefilter/filtered": 180, + "item_scraped_count": 140, +} +stats_dump_2 = { + "log_count/INFO": 12, + "log_count/WARNING": 1, + "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170), + "scheduler/enqueued/memory": 337, + "scheduler/enqueued": 337, + "scheduler/dequeued/memory": 280, + "scheduler/dequeued": 280, + "downloader/request_count": 280, + "downloader/request_method_count/GET": 280, + "downloader/request_bytes": 95754, + "downloader/response_count": 264, + "downloader/response_status_count/200": 264, + "downloader/response_bytes": 1046274, + "httpcompression/response_bytes": 5614484, + "httpcompression/response_count": 264, + "response_received_count": 264, + "request_depth_max": 16, + "dupefilter/filtered": 320, + "item_scraped_count": 248, +} + + +class TestExtPeriodicLog(PeriodicLog): + def set_a(self): + self.stats._stats = stats_dump_1 + + def set_b(self): + self.stats._stats = stats_dump_2 + + +def extension(settings=None): + crawler = Crawler(MetaSpider, settings=settings) + crawler._apply_settings() + return TestExtPeriodicLog.from_crawler(crawler) + + +class TestPeriodicLog(unittest.TestCase): + def test_extension_enabled(self): + # Expected that settings for this extension loaded succesfully + # And on certain conditions - extension raising NotConfigured + + # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} + # due to TypeError exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) + + # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} + # due to JSONDecodeError(ValueError) exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) + + # The ame for PERIODIC_LOG_DELTA: + assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) + assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + + def test_log_delta(self): + def emulate(settings=None): + spider = MetaSpider() + ext = extension(settings) + ext.spider_opened(spider) + ext.set_a() + a = ext.log_delta() + ext.set_a() + b = ext.log_delta() + ext.spider_closed(spider, reason="finished") + return ext, a, b + + def check(settings: dict, condition: typing.Callable): + ext, a, b = emulate(settings) + assert list(a["delta"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + assert list(b["delta"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + + # Including all + check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))) + + # include: + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}}, + lambda k, v: isinstance(v, (int, float)) and "downloader/" in k, + ) + + # include multiple + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" in k or "scheduler/" in k), + ) + + # exclude + check( + {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}}, + lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k, + ) + + # exclude multiple + check( + {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" not in k and "scheduler/" not in k), + ) + + # include exclude combined + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" in k and "bytes" not in k), + ) + + def test_log_stats(self): + def emulate(settings=None): + spider = MetaSpider() + ext = extension(settings) + ext.spider_opened(spider) + ext.set_a() + a = ext.log_crawler_stats() + ext.set_a() + b = ext.log_crawler_stats() + ext.spider_closed(spider, reason="finished") + return ext, a, b + + def check(settings: dict, condition: typing.Callable): + ext, a, b = emulate(settings) + assert list(a["stats"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + assert list(b["stats"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + + # Including all + check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) + + # include: + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}}, + lambda k, v: "downloader/" in k, + ) + + # include multiple + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}}, + lambda k, v: "downloader/" in k or "scheduler/" in k, + ) + + # exclude + check( + {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}}, + lambda k, v: "downloader/" not in k, + ) + + # exclude multiple + check( + {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}}, + lambda k, v: "downloader/" not in k and "scheduler/" not in k, + ) + + # include exclude combined + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, + lambda k, v: "downloader/" in k and "bytes" not in k, + ) + # diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b1059099a..89169fd7c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -36,6 +36,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FeedSlot, @@ -236,8 +237,10 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - def test_parse_credentials(self): + def setUp(self): skip_if_no_boto() + + def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", "AWS_SECRET_ACCESS_KEY": "settings_secret", @@ -273,8 +276,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - skip_if_no_boto() - settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -286,30 +287,39 @@ class S3FeedStorageTest(unittest.TestCase): verifyObject(IFeedStorage, storage) file = mock.MagicMock() - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) + if IS_BOTO3_AVAILABLE: + storage.s3_client = mock.MagicMock() yield storage.store(file) - - stub.assert_no_pending_responses() self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), ) + else: + from botocore.stub import Stubber + + with Stubber(storage.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "Body": file, + "Bucket": bucket, + "Key": key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + mock.call.close(), + ], + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -336,6 +346,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + def test_from_crawler_without_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -364,6 +387,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + def test_from_crawler_with_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -391,9 +428,22 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + @defer.inlineCallbacks - def test_store_botocore_without_acl(self): - skip_if_no_boto() + def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -405,11 +455,18 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1]) + if IS_BOTO3_AVAILABLE: + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + else: + acl = storage.s3_client.put_object.call_args[1].get("ACL") + self.assertIsNone(acl) @defer.inlineCallbacks - def test_store_botocore_with_acl(self): - skip_if_no_boto() + def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -419,9 +476,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertEqual( - storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl" - ) + if IS_BOTO3_AVAILABLE: + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + else: + acl = storage.s3_client.put_object.call_args[1]["ACL"] + self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): with LogCapture() as log: @@ -726,10 +785,9 @@ class FeedExportTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[feed_options["format"]] = Path(file_path).read_bytes() + content[feed_options["format"]] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -889,15 +947,10 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_stats_multiple_file(self): settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", "FEEDS": { printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, - "s3://bucket/key/foo.csv": { - "format": "csv", - }, "stdout:": { "format": "xml", }, @@ -909,18 +962,12 @@ class FeedExportTest(FeedExportTestBase): self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) - self.assertIn( - "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats() - ) self.assertIn( "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1 - ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 ) @@ -946,9 +993,10 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": { self._random_temp_filename(): {"format": fmt}, }, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) - self.assertEqual(b"", data[fmt]) + self.assertEqual(None, data[fmt]) @defer.inlineCallbacks def test_start_finish_exporting_items(self): @@ -1064,8 +1112,7 @@ class FeedExportTest(FeedExportTestBase): with LogCapture() as log: yield self.exported_no_data(settings) - print(log) - self.assertEqual(str(log).count("Storage.store is called"), 3) + self.assertEqual(str(log).count("Storage.store is called"), 0) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -1271,6 +1318,17 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) + @defer.inlineCallbacks + def test_export_tuple(self): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux"}, + ] + + settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} + rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] + yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + @defer.inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields @@ -1732,10 +1790,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[str(file_path)] = Path(file_path).read_bytes() + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -2236,6 +2293,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue for file in sorted(dir_name.iterdir()): content[feed["format"]].append(file.read_bytes()) finally: @@ -2419,10 +2479,11 @@ class BatchDeliveriesTest(FeedExportTestBase): / self._file_mark: {"format": fmt}, }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(b"", data[fmt][0]) + self.assertEqual(0, len(data[fmt])) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -2536,9 +2597,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) - @pytest.mark.skipif( - sys.platform == "win32", reason="Odd behaviour on file creation/output" - ) @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2560,7 +2618,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len([_ for _ in data["json"] if _])) + self.assertEqual(len(items), len(data["json"])) @defer.inlineCallbacks def test_stats_batch_file_success(self): @@ -2587,7 +2645,6 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() - bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2647,7 +2704,7 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(TestSpider, settings) yield crawler.crawl() - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() @@ -2752,196 +2809,30 @@ class FeedExportInitTest(unittest.TestCase): with self.assertRaises(NotConfigured): FeedExporter.from_crawler(crawler) - -class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - with pytest.warns( - ScrapyDeprecationWarning, - match="StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class FileFeedStorageWithoutFeedOptions(FileFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def test_init(self): - with tempfile.NamedTemporaryFile() as temp: - settings_dict = { - "FEED_URI": f"file:///{temp.name}", - "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions}, + def test_absolute_pathlib_as_uri(self): + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + Path(tmp.name).resolve(): { + "format": "json", + }, + }, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - spider = scrapy.Spider("default") + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) - with pytest.warns( - ScrapyDeprecationWarning, - match="FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): - super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) - - -class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - -class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions}, + def test_relative_pathlib_as_uri(self): + settings = { + "FEEDS": { + Path("./items.json"): { + "format": "json", + }, + }, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - -class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): - def __init__(self, uri, use_active_mode=False): - super().__init__(uri) - - -class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - -class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - maxDiff = None - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) class URIParamsTest: @@ -2990,10 +2881,7 @@ class URIParamsTest: spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with pytest.warns( - ScrapyDeprecationWarning, match="Modifying the params dictionary in-place" - ): - feed_exporter.open_spider(spider) + feed_exporter.open_spider(spider) self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4709007a8..8fdf3d56f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -275,7 +275,9 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - content_length = int(response.headers.get("Content-Length")) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) d = self.make_request(request) @@ -320,11 +322,15 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - content_length = int(response.headers.get("Content-Length")) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) # Parse the body - content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") body = json.loads(str(response.body, content_encoding)) self.assertIn("request-body", body) self.assertIn("extra-data", body) @@ -562,7 +568,9 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f"/query-params?{urlencode(params)}")) def assert_query_params(response: Response): - content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) @@ -592,6 +600,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + assert response.certificate # typing self.assertIsNotNone(response.certificate.original) self.assertEqual( response.certificate.getIssuer(), self.client_certificate.getIssuer() diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2bd68f846..6dc9ec8b7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import re import unittest import warnings import xmlrpc.client +from typing import Any, Dict, List from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse @@ -21,8 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers = {} - default_meta = {} + default_headers: Dict[bytes, List[bytes]] = {} + default_meta: Dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 54f0461e8..80d46274b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -844,7 +844,7 @@ class TextResponseTest(BaseResponseTest): with mock.patch("json.loads") as mock_json: for _ in range(2): json_response.json() - mock_json.assert_called_once_with(json_body.decode()) + mock_json.assert_called_once_with(json_body) class HtmlResponseTest(TextResponseTest): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e1ec19601..18e9608c1 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,6 +1,7 @@ import pickle import re import unittest +from typing import Optional from packaging.version import Version from pytest import mark @@ -15,7 +16,7 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: class LinkExtractorTestCase(unittest.TestCase): - extractor_cls = None + extractor_cls: Optional[type] = None def setUp(self): body = get_testdata("link_extractor", "linkextractor.html") @@ -235,20 +236,20 @@ class Base: url2 = "http://evenmorestuff.com/uglystuff/index" lx = self.extractor_cls(allow=(r"stuff1",)) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + self.assertTrue(lx.matches(url1)) + self.assertFalse(lx.matches(url2)) lx = self.extractor_cls(deny=(r"uglystuff",)) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + self.assertTrue(lx.matches(url1)) + self.assertFalse(lx.matches(url2)) lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) + self.assertFalse(lx.matches(url1)) + self.assertTrue(lx.matches(url2)) lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) + self.assertFalse(lx.matches(url1)) + self.assertTrue(lx.matches(url2)) lx = self.extractor_cls( allow=["blah1"], @@ -256,10 +257,10 @@ class Base: allow_domains=["blah1.com"], deny_domains=["blah2.com"], ) - self.assertEqual(lx.matches("http://blah1.com/blah1"), True) - self.assertEqual(lx.matches("http://blah1.com/blah2"), False) - self.assertEqual(lx.matches("http://blah2.com/blah1"), False) - self.assertEqual(lx.matches("http://blah2.com/blah2"), False) + self.assertTrue(lx.matches("http://blah1.com/blah1")) + self.assertFalse(lx.matches("http://blah1.com/blah2")) + self.assertFalse(lx.matches("http://blah2.com/blah1")) + self.assertFalse(lx.matches("http://blah2.com/blah2")) def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) diff --git a/tests/test_loader.py b/tests/test_loader.py index 0dc2de172..b0b7f8723 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,5 +1,6 @@ import dataclasses import unittest +from typing import Optional import attr from itemadapter import ItemAdapter @@ -87,7 +88,7 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class = None + item_class: Optional[type] = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 3eae87599..d7f773d5c 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -4,7 +4,6 @@ Once we remove the references from scrapy, we can remove these tests. """ import unittest -import warnings from functools import partial from itemloaders.processors import ( @@ -18,9 +17,6 @@ from itemloaders.processors import ( from scrapy.item import Field, Item from scrapy.loader import ItemLoader -from scrapy.loader.common import wrap_loader_context -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import extract_regex # test items @@ -722,24 +718,5 @@ class FunctionProcessorTestCase(unittest.TestCase): self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) -class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): - def test_deprecated_wrap_loader_context(self): - def function(*args): - return None - - with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context={}) - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - def test_deprecated_extract_regex(self): - with warnings.catch_warnings(record=True) as w: - extract_regex(r"\w+", "this is a test") - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_mail.py b/tests/test_mail.py index bc7298e9d..2535e58db 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import unittest from email.charset import Charset from io import BytesIO @@ -113,7 +111,7 @@ class MailSenderTest(unittest.TestCase): msg = self.catched_msg["msg"] self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_payload(), body) + self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body) self.assertEqual(msg.get_charset(), Charset("utf-8")) self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cb9464949..ed8483483 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from typing import Optional, Set from testfixtures import LogCapture from twisted.internet import defer @@ -54,7 +55,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums = { + expected_checksums: Optional[Set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", @@ -193,6 +194,7 @@ class FileDownloadCrawlTestCase(TestCase): ) +skip_pillow: Optional[str] try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c80666586..e7000e314 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -32,6 +33,7 @@ from scrapy.utils.test import ( get_gcs_content_and_delete, skip_if_no_boto, ) +from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func @@ -225,12 +227,16 @@ class FilesPipelineTestCase(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: + def setUp(self): + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + def test_item_fields_default(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings({"FILES_STORE": "s3://example/files/"}) - ) + pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] @@ -245,7 +251,7 @@ class FilesPipelineTestCaseFieldsMixin: pipeline = FilesPipeline.from_settings( Settings( { - "FILES_STORE": "s3://example/files/", + "FILES_STORE": self.tempdir, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", } @@ -303,11 +309,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls = attr.ib(default=lambda: []) - files = attr.ib(default=lambda: []) + file_urls: List[str] = attr.ib(default=lambda: []) + files: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls = attr.ib(default=lambda: []) - custom_files = attr.ib(default=lambda: []) + custom_file_urls: List[str] = attr.ib(default=lambda: []) + custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( @@ -461,8 +467,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.files_result_field, "this") - self.assertEqual(pipeline.files_urls_field, "that") + self.assertEqual( + pipeline.files_result_field, + UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, @@ -639,31 +650,29 @@ class TestGCSFilesStore(unittest.TestCase): class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - uri = os.environ.get("FTP_TEST_FILE_URI") - if not uri: - raise unittest.SkipTest("No FTP URI available for testing") data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} path = "full/filename" - store = FTPFilesStore(uri) - empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) - yield store.persist_file(path, buf, info=None, meta=meta, headers=None) - stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") - path = f"{store.basedir}/{path}" - content = get_ftp_content_and_delete( - path, - store.host, - store.port, - store.username, - store.password, - store.USE_ACTIVE_MODE, - ) - self.assertEqual(data.decode(), content) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + empty_dict = yield store.stat_file(path, info=None) + self.assertEqual(empty_dict, {}) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + stat = yield store.stat_file(path, info=None) + self.assertIn("last_modified", stat) + self.assertIn("checksum", stat) + self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + path = f"{store.basedir}/{path}" + content = get_ftp_content_and_delete( + path, + store.host, + store.port, + store.username, + store.password, + store.USE_ACTIVE_MODE, + ) + self.assertEqual(data, content) class ItemWithFiles(Item): @@ -682,7 +691,3 @@ def _prepare_request_object(item_url, flags=None): item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6fd8e6308..2e2e06b89 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List, Optional from unittest.mock import patch import attr @@ -18,6 +19,7 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes +skip_pillow: Optional[str] try: from PIL import Image except ImportError: @@ -26,7 +28,7 @@ except ImportError: ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} - if not encoders.issubset(set(Image.core.__dict__)): + if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] skip_pillow = "Missing JPEG encoders" else: skip_pillow = None @@ -404,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls = attr.ib(default=lambda: []) - images = attr.ib(default=lambda: []) + image_urls: List[str] = attr.ib(default=lambda: []) + images: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls = attr.ib(default=lambda: []) - custom_images = attr.ib(default=lambda: []) + custom_image_urls: List[str] = attr.ib(default=lambda: []) + custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( @@ -598,8 +600,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.images_result_field, "something_else") - self.assertEqual(pipeline.images_urls_field, "something") + self.assertEqual( + pipeline.images_result_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.images_urls_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of ImagePipeline, @@ -640,7 +648,3 @@ def _create_image(format, *a, **kw): Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf), buf - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d655eb128..820484565 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -15,7 +15,6 @@ from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -427,102 +426,6 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): return super().image_downloaded(response, request, info) -class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): - settings_dict = { - "IMAGES_STORE": "store-uri", - "IMAGES_THUMBS": {"small": (50, 50)}, - } - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) - self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) - self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(None) - self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[]) - - def _assert_method_called_with_warnings(self, method, message, warnings): - self.assertIn(method, self.pipe._mockcalled) - warningShown = False - for warning in warnings: - if ( - warning["message"] == message - and warning["category"] == ScrapyDeprecationWarning - ): - warningShown = True - self.assertTrue(warningShown) - - @inlineCallbacks - def test_media_to_download_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_to_download(self, request, info) is deprecated, " - "please use media_to_download(self, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_to_download", message, warnings) - - @inlineCallbacks - def test_media_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_downloaded(self, response, request, info) is deprecated, " - "please use media_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_downloaded", message, warnings) - - @inlineCallbacks - def test_file_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_downloaded(self, response, request, info) is deprecated, " - "please use file_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("file_downloaded", message, warnings) - - @inlineCallbacks - def test_file_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_path(self, request, response=None, info=None) is deprecated, " - "please use file_path(self, request, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("file_path", message, warnings) - - @inlineCallbacks - def test_thumb_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, " - "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("thumb_path", message, warnings) - - @inlineCallbacks - def test_get_images_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "get_images(self, response, request, info) is deprecated, " - "please use get_images(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("get_images", message, warnings) - - @inlineCallbacks - def test_image_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "image_downloaded(self, response, request, info) is deprecated, " - "please use image_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("image_downloaded", message, warnings) - - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index c05f4da91..dc0a82086 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -21,8 +21,6 @@ class MitmProxy: auth_pass = "scrapy" def start(self): - from scrapy.utils.test import get_testenv - script = """ import sys from mitmproxy.tools.main import mitmdump @@ -46,7 +44,6 @@ sys.exit(mitmdump()) "--ssl-insecure", ], stdout=PIPE, - env=get_testenv(), ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index adce6cda7..7299972f6 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,3 +1,5 @@ +from typing import List + from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -62,7 +64,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = [] + checks: List[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 8665a9205..7312eb036 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,10 +1,6 @@ -import sys import unittest -import warnings -from contextlib import suppress from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -162,30 +158,6 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): - def _assert_serializes_ok(self, request, spider=None): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - with suppress(KeyError): - del sys.modules[ - "scrapy.utils.reqser" - ] # delete module to reset the deprecation warning - - from scrapy.utils.reqser import request_from_dict as _from_dict - from scrapy.utils.reqser import request_to_dict as _to_dict - - request_copy = _from_dict(_to_dict(request, spider), spider) - self._assert_same_request(request, request_copy) - - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead", - str(caught[0].message), - ) - - class TestSpiderMixin: def __mixin_callback(self, response): pass diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 8d87a322a..d7a923085 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,3 @@ -# coding=utf-8 from twisted.trial import unittest diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index bfb370373..f8465a5ff 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,6 +2,7 @@ import collections import shutil import tempfile import unittest +from typing import Optional from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -12,6 +13,7 @@ from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -54,10 +56,11 @@ class MockCrawler(Crawler): ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) + self.stats = load_object(self.settings["STATS_CLASS"])(self) class SchedulerHandler: - priority_queue_cls = None + priority_queue_cls: Optional[str] = None jobdir = None def create_scheduler(self): @@ -251,7 +254,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue" + priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False def test_logic(self): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4a577cd8c..e7799737f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +import pytest + from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -199,6 +201,21 @@ class BaseSettingsTest(unittest.TestCase): settings.update({"key_lowprio": 3}, priority=20) self.assertEqual(settings["key_lowprio"], 1) + @pytest.mark.xfail( + raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" + ) + def test_update_kwargs(self): + settings = BaseSettings({"key": 0}) + settings.update(key=1) # pylint: disable=unexpected-keyword-arg + + @pytest.mark.xfail( + raises=AttributeError, + reason="BaseSettings.update doesn't support iterable input", + ) + def test_update_iterable(self): + settings = BaseSettings({"key": 0}) + settings.update([("key", 1)]) + def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') @@ -217,6 +234,10 @@ class BaseSettingsTest(unittest.TestCase): self.assertIn("key_highprio", settings) del settings["key_highprio"] self.assertNotIn("key_highprio", settings) + with self.assertRaises(KeyError): + settings.delete("notkey") + with self.assertRaises(KeyError): + del settings["notkey"] def test_get(self): test_configuration = { @@ -451,6 +472,31 @@ class SettingsTest(unittest.TestCase): self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) + def test_pop_item_with_default_value(self): + settings = Settings() + + with self.assertRaises(KeyError): + settings.pop("DUMMY_CONFIG") + + dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") + self.assertEqual(dummy_config_value, "dummy_value") + + def test_pop_item_with_immutable_settings(self): + settings = Settings( + {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} + ) + + self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + + settings.freeze() + + with self.assertRaises(TypeError) as error: + settings.pop("OTHER_DUMMY_CONFIG") + + self.assertEqual( + str(error.exception), "Trying to modify an immutable Settings object" + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spider.py b/tests/test_spider.py index 57c2b79cb..00da3d485 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,13 +2,16 @@ import gzip import inspect import warnings from io import BytesIO +from typing import Any from unittest import mock from testfixtures import LogCapture +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import safe_url_string from scrapy import signals +from scrapy.crawler import Crawler from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.settings import Settings @@ -96,6 +99,28 @@ class SpiderTest(unittest.TestCase): self.assertEqual(settings.get("TEST2"), "spider") self.assertEqual(settings.get("TEST3"), "project") + @inlineCallbacks + def test_settings_in_from_crawler(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + + class TestSpider(self.spider_class): + name = "test" + custom_settings = spider_settings + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("TEST1", "spider_instance", priority="spider") + return spider + + crawler = Crawler(TestSpider, project_settings) + self.assertEqual(crawler.settings.get("TEST1"), "spider") + self.assertEqual(crawler.settings.get("TEST2"), "spider") + self.assertEqual(crawler.settings.get("TEST3"), "project") + yield crawler.crawl() + self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + def test_logger(self): spider = self.spider_class("example.com") with LogCapture() as lc: diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 1d5a887cc..044455415 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,4 +1,5 @@ import logging +from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -16,7 +17,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes = set() + bypass_status_codes: Set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 2f0dda269..afffa87fb 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,4 +1,5 @@ import warnings +from typing import Any, Dict, List, Optional, Tuple from unittest import TestCase from urllib.parse import urlparse @@ -31,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta = {} - resp_headers = {} - settings = {} - scenarii = [ + req_meta: Dict[str, Any] = {} + resp_headers: Dict[str, str] = {} + settings: Dict[str, Any] = {} + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -64,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -85,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -95,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -177,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -246,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -270,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -298,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -405,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -517,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -920,7 +921,9 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii = [ + scenarii: List[ + Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent "http://scrapytest.org/2", # target diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f645f4cce..f97125b76 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ import shutil -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from twisted.trial import unittest @@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase): Path(jobdir).mkdir() try: spider = Spider(name="default") - dt = datetime.now() + dt = datetime.now(tz=timezone.utc) ss = SpiderState(jobdir) ss.spider_opened(spider) diff --git a/tests/test_stats.py b/tests/test_stats.py index 7a8adf638..3d4c7e88e 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase): @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector(self, mock_datetime): fixed_datetime = datetime(2019, 12, 1, 11, 38) - mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime) + mock_datetime.now = mock.Mock(return_value=fixed_datetime) self.crawler.stats = StatsCollector(self.crawler) ext = CoreStats.from_crawler(self.crawler) ext.spider_opened(self.spider) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 01d0ee043..65e352053 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -4,6 +4,7 @@ from unittest import TestCase from pytest import mark +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -29,6 +30,8 @@ class AsyncioTest(TestCase): assert original_reactor == reactor + @mark.only_asyncio() + @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop() is asyncio.get_running_loop() + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 78ed9a7c9..dc3f01d57 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,6 +1,8 @@ import unittest import warnings +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -21,44 +23,45 @@ class BuildComponentListTest(unittest.TestCase): def test_backward_compatible_build_dict(self): base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} custom = {"two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(base, custom, convert=lambda x: x), + ["one", "four", "five", "three"], + ) def test_return_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, convert=lambda x: x), custom + ) def test_map_dict(self): custom = {"one": 1, "two": 2, "three": 3} - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list({}, custom, convert=lambda x: x.upper()), + ["ONE", "TWO", "THREE"], + ) def test_map_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] + ) def test_duplicate_components_in_dict(self): duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - self.assertRaises( - ValueError, - build_component_list, - {}, - duplicate_dict, - convert=lambda x: x.lower(), - ) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ["a", "b", "a"] with self.assertRaises(ValueError) as cm: - build_component_list(None, duplicate_list, convert=lambda x: x) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list(None, duplicate_list, convert=lambda x: x) self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): @@ -76,9 +79,8 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - self.assertRaises( - ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower() - ) + with self.assertRaises(ValueError): + build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): # work well with None and numeric values @@ -92,15 +94,9 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) # raise exception for invalid values d = {"one": "5"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "1.0"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": [1, 2, 3]} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": {"a": "a", "b": 2}} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "lorem ipsum"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b6a84ee91..9e5f88f48 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,9 +1,13 @@ import copy import unittest +import warnings from collections.abc import Mapping, MutableMapping +from typing import Iterator +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.datatypes import ( + CaseInsensitiveDict, CaselessDict, LocalCache, LocalWeakReferencedCache, @@ -14,16 +18,16 @@ from scrapy.utils.python import garbage_collect __doctests__ = ["scrapy.utils.datatypes"] -class CaselessDictTest(unittest.TestCase): +class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {"red": 1, "black": 3} - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -42,7 +46,7 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -67,12 +71,12 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMutableMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_caseless(self): - d = CaselessDict() + d = self.dict_class() d["key_Lower"] = 1 self.assertEqual(d["KEy_loWer"], 1) self.assertEqual(d.get("KEy_loWer"), 1) @@ -82,7 +86,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key_Lower"), 3) def test_delete(self): - d = CaselessDict({"key_lower": 1}) + d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] self.assertRaises(KeyError, d.__getitem__, "key_LOWER") self.assertRaises(KeyError, d.__getitem__, "key_lower") @@ -107,15 +111,15 @@ class CaselessDictTest(unittest.TestCase): def test_fromkeys(self): keys = ("a", "b") - d = CaselessDict.fromkeys(keys) + d = self.dict_class.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) - d = CaselessDict.fromkeys(keys, 1) + d = self.dict_class.fromkeys(keys, 1) self.assertEqual(d["A"], 1) self.assertEqual(d["B"], 1) - instance = CaselessDict() + instance = self.dict_class() d = instance.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) @@ -125,31 +129,35 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d["B"], 1) def test_contains(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 - assert "a" in d + assert "A" in d def test_pop(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) self.assertRaises(KeyError, d.pop, "A") def test_normkey(self): - class MyDict(CaselessDict): - def normkey(self, key): + class MyDict(self.dict_class): + def _normkey(self, key): return key.title() + normkey = _normkey # deprecated CaselessDict class + d = MyDict() d["key-one"] = 2 self.assertEqual(list(d.keys()), ["Key-One"]) def test_normvalue(self): - class MyDict(CaselessDict): - def normvalue(self, value): + class MyDict(self.dict_class): + def _normvalue(self, value): if value is not None: return value + 1 + normvalue = _normvalue # deprecated CaselessDict class + d = MyDict({"key": 1}) self.assertEqual(d["key"], 2) self.assertEqual(d.get("key"), 2) @@ -174,11 +182,51 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key"), 2) def test_copy(self): - h1 = CaselessDict({"header1": "value"}) + h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) + assert isinstance(h2, self.dict_class) self.assertEqual(h1, h2) self.assertEqual(h1.get("header1"), h2.get("header1")) - assert isinstance(h2, CaselessDict) + self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + h3 = h1.copy() + assert isinstance(h3, self.dict_class) + self.assertEqual(h1, h3) + self.assertEqual(h1.get("header1"), h3.get("header1")) + self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + + +class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaseInsensitiveDict + + def test_repr(self): + d1 = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d1), "") + d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + self.assertEqual( + repr(d2), "" + ) + + def test_iter(self): + d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + iterkeys = iter(d) + self.assertIsInstance(iterkeys, Iterator) + self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + + +class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaselessDict + + def test_deprecation_message(self): + with warnings.catch_warnings(record=True) as caught: + self.dict_class({"foo": "bar"}) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + str(caught[0].message), + ) class SequenceExcludeTest(unittest.TestCase): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 2d9210410..eedb6f6af 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -296,3 +296,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath("scrapy.unmatched.Path") self.assertEqual(output, "scrapy.unmatched.Path") self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 6b2a458bc..7b7a25db8 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase): def test_gunzip_no_gzip_file_raises(self): self.assertRaises( - IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() + OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() ) def test_gunzip_truncated_short(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index faf7d2709..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) + self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80e15a60f..661fb47a3 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,16 +1,13 @@ import unittest -import warnings from pathlib import Path from urllib.parse import urlparse -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_base_url, get_meta_refresh, open_in_browser, - response_httprepr, response_status_message, ) @@ -20,35 +17,6 @@ __doctests__ = ["scrapy.utils.response"] class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") - def test_response_httprepr(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), b"HTTP/1.1 200 OK\r\n\r\n") - - r1 = Response( - "http://www.example.com", - status=404, - headers={"Content-type": "text/html"}, - body=b"Some body", - ) - self.assertEqual( - response_httprepr(r1), - b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body", - ) - - r1 = Response( - "http://www.example.com", - status=6666, - headers={"Content-type": "text/html"}, - body=b"Some body", - ) - self.assertEqual( - response_httprepr(r1), - b"HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body", - ) - def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" body = b" test page test body " diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 5cdcc7f7c..055db4e5b 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -58,7 +58,7 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - def test_encode_dataclass_item(self): + def test_encode_dataclass_item(self) -> None: @dataclasses.dataclass class TestDataClass: name: str diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 65522f0fd..59a95b0e2 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -364,7 +364,7 @@ for k, args in enumerate( setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate( +for k, skip_args in enumerate( [ ( r"C:\absolute\path\to\a\file.html", @@ -374,7 +374,7 @@ for k, args in enumerate( ], start=1, ): - t_method = create_skipped_scheme_t(args) + t_method = create_skipped_scheme_t(skip_args) t_method.__name__ = f"test_uri_skipped_{k:03}" setattr(GuessSchemeTest, t_method.__name__, t_method) diff --git a/tox.ini b/tox.ini index af8f1f57a..932c0b805 100644 --- a/tox.ini +++ b/tox.ini @@ -16,10 +16,6 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' - # Extras - botocore>=1.4.87 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -37,15 +33,18 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.2.0 + mypy==1.6.1 + typing-extensions==4.8.0 types-attrs==19.1.0 - types-lxml==2023.3.28 - types-Pillow==9.4.0.19 - types-Pygments==2.14.0.7 - types-pyOpenSSL==23.1.0.1 - types-setuptools==67.6.0.7 + types-lxml==2023.10.21 + types-Pillow==10.1.0.0 + types-Pygments==2.16.0.0 + types-pyOpenSSL==23.3.0.0 + types-setuptools==68.2.0.0 + # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 + w3lib >= 2.1.2 commands = - mypy --show-error-codes {posargs: scrapy tests} + mypy {posargs: scrapy tests} [testenv:pre-commit] basepython = python3 @@ -58,7 +57,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.2 + pylint==3.0.1 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -66,14 +65,14 @@ commands = basepython = python3 deps = twine==4.0.2 - build==0.10.0 + build==1.0.3 commands = python -m build --sdist twine check dist/* [pinned] deps = - cryptography==3.4.6 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -85,16 +84,11 @@ deps = Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.3.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -103,7 +97,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -126,14 +120,27 @@ commands = {[pinned]commands} basepython = python3 deps = {[testenv]deps} - boto + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + boto3==1.20.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = {[pinned]commands} [testenv:asyncio] commands = @@ -193,3 +200,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}