diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4cfba674d..53e873427 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.8.0 +current_version = 2.10.0 commit = True tag = True tag_name = {new_version} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 174d245ca..3044a1af3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 8fcf90a18..c2b686628 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -8,9 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py @@ -28,19 +25,38 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3.7 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned + - python-version: 3.8.17 + env: + TOXENV: extra-deps-pinned + - python-version: 3.8.17 + env: + TOXENV: botocore-pinned - python-version: "3.11" env: TOXENV: extra-deps + - python-version: "3.11" + env: + TOXENV: botocore + + - python-version: "3.12.0-rc.1" + env: + TOXENV: py + - python-version: "3.12.0-rc.1" + env: + TOXENV: asyncio + - python-version: "3.12.0-rc.1" + env: + TOXENV: extra-deps steps: - uses: actions/checkout@v3 @@ -51,7 +67,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f60c48841..c8d1928d7 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,12 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: windows-pinned - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py @@ -23,13 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio -# no binary package for lxml for 3.11 yet -# - python-version: "3.11" -# env: -# TOXENV: py -# - python-version: "3.11" -# env: -# TOXENV: asyncio + - python-version: "3.11" + env: + TOXENV: py + - python-version: "3.11" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf8808f2..5998ebef8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,20 +5,20 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 # 6.0.0 drops Python 3.7 support + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.5 # 5.12 drops Python 3.7 support + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.13.0 + rev: 1.15.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.3.0 + - black==23.7.0 diff --git a/README.rst b/README.rst index 970bf2c35..1918850d6 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ including a list of features. Requirements ============ -* Python 3.7+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install diff --git a/conftest.py b/conftest.py index e1d4b1213..2bfa46f5a 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,10 @@ +import platform +import sys from pathlib import Path import pytest +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -14,6 +18,10 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + "tests/ftpserver.py", + "tests/mockserver.py", + "tests/pipelines.py", + "tests/spiders.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess @@ -73,6 +81,20 @@ def only_not_asyncio(request, reactor_pytest): pytest.skip("This test is only run without --reactor=asyncio") +@pytest.fixture(autouse=True) +def requires_uvloop(request): + if not request.node.get_closest_marker("requires_uvloop"): + return + if sys.implementation.name == "pypy": + pytest.skip("uvloop does not support pypy properly") + if platform.system() == "Windows": + pytest.skip("uvloop does not support Windows") + if twisted_version == Version("twisted", 21, 2, 0): + pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + if sys.version_info >= (3, 12): + pytest.skip("uvloop doesn't support Python 3.12 yet") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/contributing.rst b/docs/contributing.rst index eef92e148..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.7 use:: +the tests with Python 3.10 use:: - tox -e py37 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py37,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.7 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py37 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -322,4 +322,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/index.rst b/docs/index.rst index 5404969e0..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -222,6 +222,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -235,6 +236,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 2c2079f68..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.7+, either the CPython implementation (default) or +Scrapy requires Python 3.8+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/docs/news.rst b/docs/news.rst index 9b9eeac71..c55c0b222 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,299 @@ Release notes ============= +.. _release-2.10.0: + +Scrapy 2.10.0 (2023-08-04) +-------------------------- + +Highlights: + +- Added Python 3.12 support, dropped Python 3.7 support. + +- The new add-ons framework simplifies configuring 3rd-party components that + support it. + +- Exceptions to retry can now be configured. + +- Many fixes and improvements for feed exports. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.7. (:issue:`5953`) + +- Added support for the upcoming Python 3.12. (:issue:`5984`) + +- Minimum versions increased for these dependencies: + + - lxml_: 4.3.0 → 4.4.1 + + - cryptography_: 3.4.6 → 36.0.0 + +- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) + +- boto3_ is now recommended instead of botocore_ for exporting to S3. + (:issue:`5833`). + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, + :issue:`5847`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter, deprecated + in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed. + (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``, + ``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``, + and ``MarshalLifoDiskQueueNonRequest``, deprecated in + Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`) + +- The property ``open_spiders`` and the methods ``has_capacity`` and + ``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`, + deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`) + +- Passing a ``spider`` argument to the + :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`, + :meth:`~scrapy.core.engine.ExecutionEngine.crawl` and + :meth:`~scrapy.core.engine.ExecutionEngine.download` methods of + :class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is + no longer supported. (:issue:`5994`, :issue:`5998`) + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use + :class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead. + (:issue:`5146`) + +- Passing the ``custom`` argument to + :func:`scrapy.utils.conf.build_component_list` is deprecated, it was used + in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy + uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same. + Code that uses this argument and cannot be switched to ``getwithbase()`` + can be switched to merging the values explicitly. (:issue:`5726`, + :issue:`5923`) + +New features +~~~~~~~~~~~~ + +- Added support for :ref:`Scrapy add-ons `. (:issue:`5950`) + +- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which + exceptions will be retried by + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`2701`, :issue:`5929`) + +- Added the possiiblity to close the spider if no items were produced in the + specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + (:issue:`5979`) + +- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports. + (:issue:`5980`) + +- Added support for using :class:`pathlib.Path` objects that refer to + absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`) + +Bug fixes +~~~~~~~~~ + +- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``. + (:issue:`872`, :issue:`5847`) + +- Fixed using absolute Windows paths when specifying output files. + (:issue:`5969`, :issue:`5971`) + +- Fixed problems with uploading large files to S3 by switching to multipart + uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`) + +- Fixed the JSON exporter writing extra commas when some exceptions occur. + (:issue:`3090`, :issue:`5952`) + +- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`, + :issue:`5705`) + +- Fixed an error when a component added by the class object throws + :exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`, + :issue:`5992`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method. + (:issue:`5959`, :issue:`5960`, :issue:`5963`) + +- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement + for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API + inconsistencies. (:issue:`5146`) + +Documentation +~~~~~~~~~~~~~ + +- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`, + :issue:`5846`) + +- Documented possible problems with early Twisted reactor installation and + their solutions. (:issue:`5981`, :issue:`6000`) + +- Added examples of making additional requests in callbacks. (:issue:`5927`) + +- Improved the feed export docs. (:issue:`5579`, :issue:`5931`) + +- Clarified the docs about request objects on redirection. (:issue:`5707`, + :issue:`5937`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added support for running tests against the installed Scrapy version. + (:issue:`4914`, :issue:`5949`) + +- Extended typing hints. (:issue:`5925`, :issue:`5977`) + +- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop`` + test. (:issue:`5951`) + +- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ`` + test on Windows. (:issue:`5847`) + +- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`) + +- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`) + +- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`) + +- Implemented cleanups. (:issue:`5965`, :issue:`5986`) + +.. _release-2.9.0: + +Scrapy 2.9.0 (2023-05-08) +------------------------- + +Highlights: + +- Per-domain download settings. +- Compatibility with new cryptography_ and new parsel_. +- JMESPath selectors from the new parsel_. +- Bug fixes. + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to + :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is + deprecated. (:issue:`5876`) + +New features +~~~~~~~~~~~~ + +- Settings correponding to :setting:`DOWNLOAD_DELAY`, + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis + via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + +- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors + available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) + +- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` + signals. (:issue:`5876`) + +- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a + curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) + +- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be + :class:`pathlib.Path` instances. (:issue:`5801`) + +Bug fixes +~~~~~~~~~ + +- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`) + +- Fixed an error when using feed postprocessing with S3 storage. + (:issue:`5500`, :issue:`5581`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method. + (:issue:`5811`, :issue:`5821`) + +- Fixed an error when using cryptography_ 40.0.0+ and + :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled. + (:issue:`5857`, :issue:`5858`) + +- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline` + for files on Google Cloud Storage are no longer Base64-encoded. + (:issue:`5874`, :issue:`5891`) + +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The :command:`parse` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The :command:`genspider` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + +- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers. + (:issue:`5914`, :issue:`5917`) + +- Fixed an error breaking user handling of send failures in + :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + +Documentation +~~~~~~~~~~~~~ + +- Expanded contributing docs. (:issue:`5109`, :issue:`5851`) + +- Added blacken-docs_ to pre-commit and reformatted the docs with it. + (:issue:`5813`, :issue:`5816`) + +- Fixed a JS issue. (:issue:`5875`, :issue:`5877`) + +- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`) + +- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`, + :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) + +- Tests for most of the examples in the docs are now run as a part of CI, + found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`) + +- Removed usage of deprecated Python classes. (:issue:`5849`) + +- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`) + +- Fixed a random failure of the ``test_feedexport.test_batch_path_differ`` + test. (:issue:`5855`, :issue:`5898`) + +- Updated docstrings to match output produced by parsel_ 1.8.1 so that they + don't cause test failures. (:issue:`5902`, :issue:`5919`) + +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`, + :issue:`5908`) + +.. _blacken-docs: https://github.com/adamchainz/blacken-docs + .. _release-2.8.0: Scrapy 2.8.0 (2023-02-02) @@ -4207,8 +4500,6 @@ Relocations + Note: telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) -.. _parsel: https://github.com/scrapy/parsel - Bugfixes ~~~~~~~~ @@ -5628,6 +5919,7 @@ First release of Scrapy. .. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ @@ -5638,6 +5930,7 @@ First release of Scrapy. .. _LevelDB: https://github.com/google/leveldb .. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel: https://github.com/scrapy/parsel .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..1bf2172bd --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,193 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. + +The ``ADDONS`` setting is a dict in which every key is an add-on class or its +import path and the value is its priority. + +This is an example where two add-ons are enabled in a project's +``settings.py``:: + + ADDONS = { + 'path.to.someaddon': 0, + SomeAddonClass: 1, + } + + +Writing your own add-ons +======================== + +Add-ons are Python classes that include the following method: + +.. method:: update_settings(settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an add-on instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access + them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this add-on + :type crawler: :class:`~scrapy.crawler.Crawler` + +The settings set by the add-on should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an add-on-specific setting that governs whether the add-on will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. + +Fallbacks +--------- + +Some components provided by add-ons need to fall back to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The add-ons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided by the add-on (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several add-ons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. + + +Add-on examples +=============== + +Set some basic configuration: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings.set("DNSCACHE_ENABLED", True, "addon") + +Check dependencies: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + try: + import boto + except ImportError: + raise NotConfigured("MyAddon requires the boto library") + ... + +Access the crawler instance: + +.. code-block:: python + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def update_settings(self, settings): + ... + +Use a fallback component: + +.. code-block:: python + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(FALLBACK_SETTING)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 268344879..16c28405c 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..07baea071 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -27,54 +27,43 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') -.. _using-custom-loops: +.. _asyncio-preinstalled-reactor: -Using custom asyncio loops -========================== +Handling a pre-installed reactor +================================ -You can also use custom asyncio event loops with the asyncio reactor. Set the -:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to -use it instead of the default asyncio event loop. +``twisted.internet.reactor`` and some other Twisted imports install the default +Twisted reactor as a side effect. Once a Twisted reactor is installed, it is +not possible to switch to a different reactor at run time. + +If you :ref:`configure the asyncio Twisted reactor ` and, at +run time, Scrapy complains that a different reactor is already installed, +chances are you have some such imports in your code. + +You can usually fix the issue by moving those offending module-level Twisted +imports to the method or function definitions where they are used. For example, +if you have something like: + +.. code-block:: python + + from twisted.internet import reactor -.. _asyncio-windows: + def my_function(): + reactor.callLater(...) -Windows-specific notes -====================== +Switch to something like: -The Windows implementation of :mod:`asyncio` can use two event loop -implementations: +.. code-block:: python -- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required - when using Twisted. + def my_function(): + from twisted.internet import reactor -- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work - with Twisted. + reactor.callLater(...) -So on Python 3.8+ the event loop class needs to be changed. - -.. versionchanged:: 2.6.0 - The event loop class is changed automatically when you change the - :setting:`TWISTED_REACTOR` setting or call - :func:`~scrapy.utils.reactor.install_reactor`. - -To change the event loop class manually, call the following code before -installing the reactor:: - - import asyncio - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -You can put this in the same function that installs the reactor, if you do that -yourself, or in some code that runs before the reactor is installed, e.g. -``settings.py``. - -.. note:: Other libraries you use may require - :class:`~asyncio.ProactorEventLoop`, e.g. because it supports - subprocesses (this is the case with `playwright`_), so you cannot use - them together with Scrapy on Windows (but you should be able to use - them on WSL or native Linux). - -.. _playwright: https://github.com/microsoft/playwright-python +Alternatively, you can try to :ref:`manually install the asyncio reactor +`, with :func:`~scrapy.utils.reactor.install_reactor`, before +those imports happen. .. _asyncio-await-dfd: @@ -122,3 +111,36 @@ example: f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) + + +.. _asyncio-windows: + +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations, :class:`~asyncio.ProactorEventLoop` (default) and +:class:`~asyncio.SelectorEventLoop`. However, only +:class:`~asyncio.SelectorEventLoop` works with Twisted. + +Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event +loop class to use it instead of the default asyncio event loop. diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1ed55f000..478dd9647 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use .. code-block:: python - from pkg_resources import parse_version + from packaging.version import parse as parse_version import scrapy diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3916bd295..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -134,6 +134,63 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + .. _sync-async-spider-middleware: Mixing synchronous and asynchronous spider middlewares diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 7665a901a..a8e5b23bf 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -915,6 +915,7 @@ settings (see the settings documentation for more info): * :setting:`RETRY_ENABLED` * :setting:`RETRY_TIMES` * :setting:`RETRY_HTTP_CODES` +* :setting:`RETRY_EXCEPTIONS` .. reqmeta:: dont_retry @@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_EXCEPTIONS + +RETRY_EXCEPTIONS +^^^^^^^^^^^^^^^^ + +Default:: + + [ + 'twisted.internet.defer.TimeoutError', + 'twisted.internet.error.TimeoutError', + 'twisted.internet.error.DNSLookupError', + 'twisted.internet.error.ConnectionRefusedError', + 'twisted.internet.error.ConnectionDone', + 'twisted.internet.error.ConnectError', + 'twisted.internet.error.ConnectionLost', + 'twisted.internet.error.TCPTimedOutError', + 'twisted.web.client.ResponseFailed', + IOError, + 'scrapy.core.downloader.handlers.http11.TunnelError', + ] + +List of exceptions to retry. + +Each list entry may be an exception type or its import path as a string. + +An exception will not be caught when the exception type is not in +:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request +has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught +exception propagation, see +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index ae94c55a4..b0b98172f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following settings: * :setting:`CLOSESPIDER_TIMEOUT` +* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` @@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by timeout. +.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM + +CLOSESPIDER_TIMEOUT_NO_ITEM +""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies a number of seconds. If the spider has not produced +any items in the last number of seconds, it will be closed with the reason +``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed +regardless if it hasn't produced any items. + .. setting:: CLOSESPIDER_ITEMCOUNT CLOSESPIDER_ITEMCOUNT diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..700775e4b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -101,12 +101,12 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` - :ref:`topics-feed-storage-ftp` -- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-s3` (requires boto3_) - :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the botocore_ +not available. For example, the S3 backend is only available if the boto3_ library is installed. @@ -156,8 +156,8 @@ The feeds are stored in the local filesystem. - Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if -you specify an absolute path like ``/tmp/export.csv``. This only works on Unix -systems though. +you specify an absolute path like ``/tmp/export.csv`` (Unix systems only). +Alternatively you can also use a :class:`pathlib.Path` object. .. _topics-feed-storage-ftp: @@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -193,7 +199,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `botocore`_ >= 1.4.87 +- Required external libraries: `boto3`_ >= 1.20.0 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -204,10 +210,18 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys -You can also define a custom ACL and custom endpoint for exported feeds using this setting: +You can also define a custom ACL, custom endpoint, and region name for exported +feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +- :setting:`AWS_REGION_NAME` + +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. This storage backend uses :ref:`delayed file delivery `. @@ -236,6 +250,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python @@ -488,6 +508,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported `_) + - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) .. versionadded:: 2.4.0 @@ -552,9 +574,12 @@ to ``.json`` or ``.xml``. FEED_STORE_EMPTY ---------------- -Default: ``False`` +Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option +` is enabled. .. setting:: FEED_STORAGES @@ -779,6 +804,6 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ -.. _botocore: https://github.com/boto/botocore +.. _boto3: https://github.com/boto/boto3 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index bc26bbebe..a5f6e07b8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -215,7 +215,7 @@ item. screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) response = await maybe_deferred_to_future( - spider.crawler.engine.download(request, spider) + spider.crawler.engine.download(request) ) if response.status != 200: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 407df32d2..41df51589 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1103,9 +1103,10 @@ Response objects through all :ref:`Downloader Middlewares `. In particular, this means that: - - HTTP redirections will cause the original request (to the URL before - redirection) to be assigned to the redirected response (with the final - URL after redirection). + - HTTP redirections will create a new request from the request before + redirection. It has the majority of the same metadata and original + request attributes and gets assigned to the redirected response + instead of the propagation of the original request. - Response.request.url doesn't always equal Response.url diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4412b5c1c..e1936eb5b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by add-ons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -66,8 +67,8 @@ Example:: ---------------------- Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +own settings that will take precedence and override the project ones. One way +to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: .. code-block:: python @@ -81,6 +82,22 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } +It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, +and settings set there should use the "spider" priority explicitly: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def update_settings(cls, settings): + super().update_settings(settings) + settings.set("SOME_SETTING", "some value", priority="spider") + 3. Project settings module -------------------------- @@ -89,7 +106,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by add-ons +-------------------------- + +:ref:`Add-ons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +120,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` @@ -201,6 +224,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``{}`` + +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -783,7 +816,7 @@ DOWNLOAD_SLOTS Default: ``{}`` -Allows to define concurrency/delay parameters on per slot(domain) basis: +Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python @@ -964,7 +997,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 788bd7678..5c3bf6e72 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -145,6 +145,46 @@ scrapy.Spider :param kwargs: keyword arguments passed to the :meth:`__init__` method :type kwargs: dict + .. classmethod:: update_settings(settings) + + The ``update_settings()`` method is used to modify the spider's settings + and is called during initialization of a spider instance. + + It takes a :class:`~scrapy.settings.Settings` object as a parameter and + can add or update the spider's configuration values. This method is a + class method, meaning that it is called on the :class:`~scrapy.Spider` + class and allows all instances of the spider to share the same + configuration. + + While per-spider settings can be set in + :attr:`~scrapy.Spider.custom_settings`, using ``update_settings()`` + allows you to dynamically add, remove or change settings based on other + settings, spider attributes or other factors and use setting priorities + other than ``'spider'``. Also, it's easy to extend ``update_settings()`` + in a subclass by overriding it, while doing the same with + :attr:`~scrapy.Spider.custom_settings` can be hard. + + For example, suppose a spider needs to modify :setting:`FEEDS`: + + .. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + custom_feed = { + "/home/user/documents/items.json": { + "format": "json", + "indent": 4, + } + } + + @classmethod + def update_settings(cls, settings): + super().update_settings(settings) + settings.setdefault("FEEDS", {}).update(cls.custom_feed) + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 1f270837c..c17b9d511 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -30,7 +30,7 @@ def main(): try: with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() - except IOError: + except OSError: print("linkcheck output not found; please run linkcheck first.") sys.exit(1) diff --git a/pytest.ini b/pytest.ini index 866f0c950..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed + requires_uvloop: marks tests as only enabled when uvloop is known to be working filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/scrapy/VERSION b/scrapy/VERSION index 834f26295..10c2c0c3d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.8.0 +2.10.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a757a9290..cc0e539c4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 7): - print(f"Scrapy {__version__} requires Python 3.7+") +if sys.version_info < (3, 8): + print(f"Scrapy {__version__} requires Python 3.8+") sys.exit(1) diff --git a/scrapy/addons.py b/scrapy/addons.py new file mode 100644 index 000000000..02dd4fde8 --- /dev/null +++ b/scrapy/addons.py @@ -0,0 +1,54 @@ +import logging +from typing import TYPE_CHECKING, Any, List + +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings +from scrapy.utils.conf import build_component_list +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + +logger = logging.getLogger(__name__) + + +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" + + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler + self.addons: List[Any] = [] + + def load_settings(self, settings: Settings) -> None: + """Load add-ons and configurations from a settings object. + + This will load the add-on for every add-on path in the + ``ADDONS`` setting and execute their ``update_settings`` methods. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + enabled: List[Any] = [] + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + logger.warning( + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": enabled, + }, + extra={"crawler": self.crawler}, + ) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 730e55350..6580ba9ce 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,8 +3,7 @@ import cProfile import inspect import os import sys - -import pkg_resources +from importlib.metadata import entry_points import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -49,7 +48,11 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in pkg_resources.iter_entry_points(group): + if sys.version_info >= (3, 10): + eps = entry_points(group=group) + else: + eps = entry_points().get(group, ()) + for entry_point in eps: obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9baee3a48..2aa569cdd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,7 +4,7 @@ Base class for Scrapy commands import argparse import os from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from twisted.python import failure @@ -116,7 +116,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1359e445f..cdb7ad4ae 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,10 @@ import sys +from argparse import Namespace +from typing import List, Type from w3lib.url import is_url +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request @@ -57,7 +60,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_): sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -73,7 +76,8 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: spidercls = spider_loader.load(opts.spider) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 63c23d04c..0a5e61f7a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,8 +3,11 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +from argparse import Namespace from threading import Thread +from typing import List, Type +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -54,15 +57,16 @@ class Command(ScrapyCommand): """ pass - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file url = guess_scheme(url) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 86098edca..1ec2a0234 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,7 +2,8 @@ import re import sys from functools import wraps from inspect import getmembers -from typing import Dict +from types import CoroutineType +from typing import AsyncGenerator, Dict from unittest import TestCase from scrapy.http import Request @@ -37,7 +38,10 @@ class Contract: else: results.addSuccess(self.testcase_pre) finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list(iterate_spider_output(cb_result)) request.callback = wrapper @@ -49,7 +53,10 @@ class Contract: @wraps(cb) def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + output = list(iterate_spider_output(cb_result)) try: results.startTest(self.testcase_post) self.post_process(output) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3e5a281b2..dad384ddc 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst """ import logging -import warnings from time import time from typing import ( TYPE_CHECKING, @@ -14,7 +13,6 @@ from typing import ( Generator, Iterable, Iterator, - List, Optional, Set, Type, @@ -29,7 +27,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, DontCloseSpider from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -213,7 +211,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request, self.spider) + d = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -266,13 +264,7 @@ class ExecutionEngine: ) return d - def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + def spider_is_idle(self) -> bool: if self.slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] @@ -285,18 +277,8 @@ class ExecutionEngine: return False return True - def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - raise RuntimeError( - f"The spider {spider.name!r} does not match the open spider" - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) @@ -311,39 +293,24 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + def download(self, request: Request) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - logger.warning( - "The spider '%s' does not match the open spider", spider.name - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request, spider).addBoth( - self._downloaded, request, spider - ) + return self._download(request).addBoth(self._downloaded, request) def _downloaded( - self, result: Union[Response, Request], request: Request, spider: Spider + self, result: Union[Response, Request], request: Request ) -> Union[Deferred, Response]: assert self.slot is not None # typing self.slot.remove_request(request) - return self.download(result, spider) if isinstance(result, Request) else result + return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: + def _download(self, request: Request) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) - if spider is None: - spider = self.spider - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError( @@ -352,15 +319,17 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - assert spider is not None - logkws = self.logformatter.crawled(result.request, result, spider) + assert self.spider is not None + logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.spider} + ) self.signals.send_catch_log( signal=signals.response_received, response=result, request=result.request, - spider=spider, + spider=self.spider, ) return result @@ -369,8 +338,8 @@ class ExecutionEngine: self.slot.nextcall.schedule() return _ - assert spider is not None - dwld = self.downloader.fetch(request, spider) + assert self.spider is not None + dwld = self.downloader.fetch(request, self.spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) return dwld @@ -485,31 +454,3 @@ class ExecutionEngine: dfd.addBoth(lambda _: self._spider_closed_callback(spider)) return dfd - - @property - def open_spiders(self) -> List[Spider]: - warnings.warn( - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return [self.spider] if self.spider is not None else [] - - def has_capacity(self) -> bool: - warnings.warn( - "ExecutionEngine.has_capacity is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return not bool(self.slot) - - def schedule(self, request: Request, spider: Spider) -> None: - warnings.warn( - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if self.slot is None: - raise RuntimeError("Engine slot not assigned") - self._schedule_request(request, spider) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3fb0bbaff..70b6dc8a1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import json import logging from abc import abstractmethod from pathlib import Path -from typing import Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred @@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. """ @@ -325,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.dqdir state = self._read_dqs_state(self.dqdir) q = create_instance( self.pqclass, diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a85f6a661..a54929712 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -364,6 +364,7 @@ class Scraper: spider=spider, exception=output.value, ) + assert ex logkws = self.logformatter.item_error(item, ex, response, spider) logger.log( *logformatter_adapter(logkws), diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 256f6e2c5..25823b6ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,9 +4,14 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast -from twisted.internet import defer +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) from zope.interface.exceptions import DoesNotImplement try: @@ -18,12 +23,13 @@ except ImportError: from zope.interface.verify import verifyClass from scrapy import Spider, signals +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( @@ -54,7 +60,7 @@ class Crawler: def __init__( self, spidercls: Type[Spider], - settings: Union[None, dict, Settings] = None, + settings: Union[None, Dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -67,6 +73,9 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) + self.addons: AddonManager = AddonManager(self) + self.addons.load_settings(self.settings) + self.signals: SignalManager = SignalManager(self) self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) @@ -118,8 +127,8 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None - @defer.inlineCallbacks - def crawl(self, *args, **kwargs): + @inlineCallbacks + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") self.crawling = True @@ -129,26 +138,27 @@ class Crawler: self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) - yield defer.maybeDeferred(self.engine.start) + yield maybeDeferred(self.engine.start) except Exception: self.crawling = False if self.engine is not None: yield self.engine.close() raise - def _create_spider(self, *args, **kwargs): + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) - def _create_engine(self): + def _create_engine(self) -> ExecutionEngine: return ExecutionEngine(self, lambda _: self.stop()) - @defer.inlineCallbacks - def stop(self): + @inlineCallbacks + def stop(self) -> Generator[Deferred, Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False - yield defer.maybeDeferred(self.engine.stop) + assert self.engine + yield maybeDeferred(self.engine.stop) class CrawlerRunner: @@ -171,7 +181,7 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings): + def _get_spider_loader(settings: BaseSettings): """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) @@ -190,26 +200,21 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[Deferred] = set() self.bootstrap_failed = False - @property - def spiders(self): - warnings.warn( - "CrawlerRunner.spiders attribute is renamed to " - "CrawlerRunner.spider_loader.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return self.spider_loader - - def crawl(self, crawler_or_spidercls, *args, **kwargs): + def crawl( + self, + crawler_or_spidercls: Union[Type[Spider], str, Crawler], + *args: Any, + **kwargs: Any, + ) -> Deferred: """ Run a crawler with the provided arguments. @@ -239,12 +244,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler, *args, **kwargs): + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result): + def _done(result: Any) -> Any: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -252,7 +257,9 @@ class CrawlerRunner: return d.addBoth(_done) - def create_crawler(self, crawler_or_spidercls): + def create_crawler( + self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -272,21 +279,22 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + # temporary cast until self.spider_loader is typed + return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self): + def stop(self) -> Deferred: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in list(self.crawlers)]) + return DeferredList([c.stop() for c in list(self.crawlers)]) - @defer.inlineCallbacks - def join(self): + @inlineCallbacks + def join(self) -> Generator[Deferred, Any, None]: """ join() @@ -294,7 +302,7 @@ class CrawlerRunner: completed their executions. """ while self._active: - yield defer.DeferredList(self._active) + yield DeferredList(self._active) class CrawlerProcess(CrawlerRunner): @@ -321,13 +329,17 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True): + def __init__( + self, + settings: Union[Dict[str, Any], Settings, None] = None, + install_root_handler: bool = True, + ): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False - def _signal_shutdown(self, signum, _): + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) @@ -338,7 +350,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._graceful_stop_reactor) - def _signal_kill(self, signum, _): + def _signal_kill(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) @@ -348,14 +360,19 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - return Crawler(spidercls, self.settings, init_reactor=init_reactor) + # temporary cast until self.spider_loader is typed + return Crawler( + cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + ) - def start(self, stop_after_crawl=True, install_signal_handlers=True): + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -389,12 +406,12 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger("before", "shutdown", self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _graceful_stop_reactor(self): + def _graceful_stop_reactor(self) -> Deferred: d = self.stop() d.addBoth(self._stop_reactor) return d - def _stop_reactor(self, _=None): + def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor try: diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 5839dc243..3b8702419 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -63,7 +63,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) @@ -72,7 +72,7 @@ class DecompressionMiddleware: def _is_bzip2(self, response): try: body = bz2.decompress(response.body) - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index b9316c43a..ac87d4a4e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -37,7 +37,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, + OSError, ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 081642a4b..50cbc3111 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +import warnings from logging import Logger, getLogger from typing import Optional, Union -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - ConnectionRefusedError, - DNSLookupError, - TCPTimedOutError, - TimeoutError, -) -from twisted.web.client import ResponseFailed - -from scrapy.core.downloader.handlers.http11 import TunnelError -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http.request import Request +from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message retry_logger = getLogger(__name__) +class BackwardsCompatibilityMetaclass(type): + @property + def EXCEPTIONS_TO_RETRY(cls): + warnings.warn( + "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " + "Use the RETRY_EXCEPTIONS setting instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return tuple( + load_object(x) if isinstance(x, str) else x + for x in Settings().getlist("RETRY_EXCEPTIONS") + ) + + def get_retry_request( request: Request, *, @@ -121,23 +126,7 @@ def get_retry_request( return None -class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to - # decompress an empty response - EXCEPTIONS_TO_RETRY = ( - defer.TimeoutError, - TimeoutError, - DNSLookupError, - ConnectionRefusedError, - ConnectionDone, - ConnectError, - ConnectionLost, - TCPTimedOutError, - ResponseFailed, - IOError, - TunnelError, - ) - +class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def __init__(self, settings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured @@ -147,6 +136,16 @@ class RetryMiddleware: ) self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + if not hasattr( + self, "EXCEPTIONS_TO_RETRY" + ): # If EXCEPTIONS_TO_RETRY is not "overriden" + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) + else: + self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY + @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) @@ -160,7 +159,7 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( + if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d796e5cbb..d2639104b 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging from pathlib import Path -from typing import Optional, Set, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Set from warnings import warn from twisted.internet.defer import Deferred @@ -10,16 +12,22 @@ from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import RequestFingerprinter, referer_str +from scrapy.utils.request import ( + RequestFingerprinter, + RequestFingerprinterProtocol, + referer_str, +) -BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler class BaseDupeFilter: @classmethod - def from_settings( - cls: Type[BaseDupeFilterTV], settings: BaseSettings - ) -> BaseDupeFilterTV: + def from_settings(cls, settings: BaseSettings) -> Self: return cls() def request_seen(self, request: Request) -> bool: @@ -36,9 +44,6 @@ class BaseDupeFilter: pass -RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") - - class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" @@ -47,10 +52,12 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter=None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> None: self.file = None - self.fingerprinter = fingerprinter or RequestFingerprinter() + self.fingerprinter: RequestFingerprinterProtocol = ( + fingerprinter or RequestFingerprinter() + ) self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug @@ -62,8 +69,11 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_settings( - cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None - ) -> RFPDupeFilterTV: + cls, + settings: BaseSettings, + *, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, + ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") try: return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @@ -75,11 +85,11 @@ class RFPDupeFilter(BaseDupeFilter): ScrapyDeprecationWarning, ) result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter + result.fingerprinter = fingerprinter or RequestFingerprinter() return result @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: try: return cls.from_settings( crawler.settings, diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fedd02805..6d188c489 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from typing import Any # Internal @@ -51,7 +52,7 @@ class StopDownload(Exception): should be handled by the request errback. Note that 'fail' is a keyword-only argument. """ - def __init__(self, *, fail=True): + def __init__(self, *, fail: bool = True): super().__init__() self.fail = fail @@ -77,7 +78,7 @@ class NotSupported(Exception): class UsageError(Exception): """To indicate a command-line usage error""" - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): self.print_help = kw.pop("print_help", True) super().__init__(*a, **kw) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index bb3e3c662..f85f1dad8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,13 +7,11 @@ import io import marshal import pickle import pprint -import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import ItemAdapter, is_item -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -133,6 +131,13 @@ class JsonItemExporter(BaseItemExporter): if self.indent is not None: self.file.write(b"\n") + def _add_comma_after_first(self): + if self.first_item: + self.first_item = False + else: + self.file.write(b",") + self._beautify_newline() + def start_exporting(self): self.file.write(b"[") self._beautify_newline() @@ -142,14 +147,10 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item): - if self.first_item: - self.first_item = False - else: - self.file.write(b",") - self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) - data = self.encoder.encode(itemdict) - self.file.write(to_bytes(data, self.encoding)) + data = to_bytes(self.encoder.encode(itemdict), self.encoding) + self._add_comma_after_first() + self.file.write(data) class XmlItemExporter(BaseItemExporter): @@ -255,6 +256,9 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def finish_exporting(self): + self.stream.detach() # Avoid closing the wrapped file. + def _build_row(self, values): for s in values: try: @@ -324,13 +328,7 @@ class PythonItemExporter(BaseItemExporter): """ def _configure(self, options, dont_fail=False): - self.binary = options.pop("binary", True) super()._configure(options, dont_fail) - if self.binary: - warnings.warn( - "PythonItemExporter will drop support for binary export in the future", - ScrapyDeprecationWarning, - ) if not self.encoding: self.encoding = "utf-8" @@ -345,18 +343,14 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): - return encode_func(value, encoding=self.encoding) + return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item): for key, value in ItemAdapter(item).items(): - key = to_bytes(key) if self.binary else key yield key, self._serialize_value(value) def export_item(self, item): result = dict(self._get_serialized_fields(item)) - if self.binary: - result = dict(self._serialize_item(result)) return result diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index bb6f832f2..4307b4170 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,11 +4,14 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +import logging from collections import defaultdict from scrapy import signals from scrapy.exceptions import NotConfigured +logger = logging.getLogger(__name__) + class CloseSpider: def __init__(self, crawler): @@ -19,6 +22,7 @@ class CloseSpider: "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), + "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), } if not any(self.close_on.values()): @@ -34,6 +38,15 @@ class CloseSpider: crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + if self.close_on.get("timeout_no_item"): + self.timeout_no_item = self.close_on["timeout_no_item"] + self.items_in_period = 0 + crawler.signals.connect( + self.spider_opened_no_item, signal=signals.spider_opened + ) + crawler.signals.connect( + self.item_scraped_no_item, signal=signals.item_scraped + ) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -69,3 +82,31 @@ class CloseSpider: task = getattr(self, "task", False) if task and task.active(): task.cancel() + + task_no_item = getattr(self, "task_no_item", False) + if task_no_item and task_no_item.running: + task_no_item.stop() + + def spider_opened_no_item(self, spider): + from twisted.internet import task + + self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item.start(self.timeout_no_item, now=False) + + logger.info( + f"Spider will stop when no items are produced after " + f"{self.timeout_no_item} seconds." + ) + + def item_scraped_no_item(self, item, spider): + self.items_in_period += 1 + + def _count_items_produced(self, spider): + if self.items_in_period >= 1: + self.items_in_period = 0 + else: + logger.info( + f"Closing spider since no items were produced in the last " + f"{self.timeout_no_item} seconds." + ) + self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 30c987253..302a615f2 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,7 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime +from datetime import datetime, timezone from scrapy import signals @@ -22,11 +22,11 @@ class CoreStats: return o def spider_opened(self, spider): - self.start_time = datetime.utcnow() + self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) def spider_closed(self, spider, reason): - finish_time = datetime.utcnow() + finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() self.stats.set_value( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bcf0b779a..4e846d1bd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -8,10 +8,10 @@ import logging import re import sys import warnings -from datetime import datetime -from pathlib import Path +from datetime import datetime, timezone +from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -33,6 +33,13 @@ from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +try: + import boto3 # noqa: F401 + + IS_BOTO3_AVAILABLE = True +except ImportError: + IS_BOTO3_AVAILABLE = False + def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): argument_names = get_func_args(builder) @@ -165,6 +172,7 @@ class S3FeedStorage(BlockingFeedStorage): *, feed_options=None, session_token=None, + region_name=None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -176,16 +184,41 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - import botocore.session + self.region_name = region_name + + if IS_BOTO3_AVAILABLE: + import boto3.session + + session = boto3.session.Session() + + self.s3_client = session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) + else: + warnings.warn( + "`botocore` usage has been deprecated for S3 feed " + "export, please use `boto3` to avoid problems", + category=ScrapyDeprecationWarning, + ) + + import botocore.session + + session = botocore.session.get_session() + + self.s3_client = session.create_client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) - session = botocore.session.get_session() - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - ) if feed_options and feed_options.get("overwrite", True) is False: logger.warning( "S3 does not support appending to files. To " @@ -203,15 +236,22 @@ class S3FeedStorage(BlockingFeedStorage): session_token=crawler.settings["AWS_SESSION_TOKEN"], acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, + region_name=crawler.settings["AWS_REGION_NAME"] or None, feed_options=feed_options, ) def _store_in_thread(self, file): file.seek(0) - kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + if IS_BOTO3_AVAILABLE: + kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) + else: + kwargs = {"ACL": self.acl} if self.acl else {} + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + ) file.close() @@ -242,15 +282,23 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False, *, feed_options=None): + def __init__( + self, + uri: str, + use_active_mode: bool = False, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): u = urlparse(uri) - self.host = u.hostname - self.port = int(u.port or "21") - self.username = u.username - self.password = unquote(u.password or "") - self.path = u.path - self.use_active_mode = use_active_mode - self.overwrite = not feed_options or feed_options.get("overwrite", True) + if not u.hostname: + raise ValueError(f"Got a storage URI without a hostname: {uri}") + self.host: str = u.hostname + self.port: int = int(u.port or "21") + self.username: str = u.username or "" + self.password: str = unquote(u.password or "") + self.path: str = u.path + self.use_active_mode: bool = use_active_mode + self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): @@ -277,8 +325,6 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - file, - exporter, storage, uri, format, @@ -286,9 +332,14 @@ class FeedSlot: batch_id, uri_template, filter, + feed_options, + spider, + exporters, + settings, + crawler, ): - self.file = file - self.exporter = exporter + self.file = None + self.exporter = None self.storage = storage # feed params self.batch_id = batch_id @@ -297,15 +348,44 @@ class FeedSlot: self.uri_template = uri_template self.uri = uri self.filter = filter + # exporter params + self.feed_options = feed_options + self.spider = spider + self.exporters = exporters + self.settings = settings + self.crawler = crawler # flags self.itemcount = 0 self._exporting = False + self._fileloaded = False def start_exporting(self): + if not self._fileloaded: + self.file = self.storage.open(self.spider) + if "postprocessing" in self.feed_options: + self.file = PostProcessingManager( + self.feed_options["postprocessing"], self.file, self.feed_options + ) + self.exporter = self._get_exporter( + file=self.file, + format=self.feed_options["format"], + fields_to_export=self.feed_options["fields"], + encoding=self.feed_options["encoding"], + indent=self.feed_options["indent"], + **self.feed_options["item_export_kwargs"], + ) + self._fileloaded = True + if not self._exporting: self.exporter.start_exporting() self._exporting = True + def _get_instance(self, objcls, *args, **kwargs): + return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + + def _get_exporter(self, file, format, *args, **kwargs): + return self._get_instance(self.exporters[format], file, *args, **kwargs) + def finish_exporting(self): if self._exporting: self.exporter.finish_exporting() @@ -347,7 +427,9 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) - uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects + uri = self.settings["FEED_URI"] + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings @@ -357,7 +439,8 @@ class FeedExporter: # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict("FEEDS").items(): - uri = str(uri) # handle pathlib.Path objects + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) @@ -406,11 +489,16 @@ class FeedExporter: return slot_.file.file return slot_.file - slot.finish_exporting() - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, get_file(slot)) + if slot.itemcount: + # Normal case + slot.finish_exporting() + elif slot.store_empty and slot.batch_id == 1: + # Need to store the empty file + slot.start_exporting() + slot.finish_exporting() + else: + # In this case, the file is not stored, so no processing is required. + return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, get_file(slot)) @@ -455,23 +543,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - file = storage.open(spider) - if "postprocessing" in feed_options: - file = PostProcessingManager( - feed_options["postprocessing"], file, feed_options - ) - - exporter = self._get_exporter( - file=file, - format=feed_options["format"], - fields_to_export=feed_options["fields"], - encoding=feed_options["encoding"], - indent=feed_options["indent"], - **feed_options["item_export_kwargs"], - ) slot = FeedSlot( - file=file, - exporter=exporter, storage=storage, uri=uri, format=feed_options["format"], @@ -479,9 +551,12 @@ class FeedExporter: batch_id=batch_id, uri_template=uri_template, filter=self.filters[uri_template], + feed_options=feed_options, + spider=spider, + exporters=self.exporters, + settings=self.settings, + crawler=getattr(self, "crawler", None), ) - if slot.store_empty: - slot.start_exporting() return slot def item_scraped(self, item, spider): @@ -553,7 +628,7 @@ class FeedExporter: def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme - if scheme in self.storages: + if scheme in self.storages or PureWindowsPath(uri).drive: try: self._get_storage(uri, feed_options) return True @@ -565,21 +640,13 @@ class FeedExporter: else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) - def _get_instance(self, objcls, *args, **kwargs): - return create_instance( - objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs - ) - - def _get_exporter(self, file, format, *args, **kwargs): - return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri, feed_options): """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that do not support it, and issuing a deprecation warning instead. """ - feedcls = self.storages[urlparse(uri).scheme] + feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): @@ -609,25 +676,16 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - utc_now = datetime.utcnow() + utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - original_params = params.copy() uripar_function = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params ) new_params = uripar_function(params, spider) - if new_params is None or original_params != params: - warnings.warn( - "Modifying the params dictionary in-place in the function defined in " - "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS " - "setting is deprecated. The function must return a new dictionary " - "instead.", - category=ScrapyDeprecationWarning, - ) return new_params if new_params is not None else params def _load_filter(self, feed_options): diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 2540be01a..822597c84 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from w3lib.http import headers_dict_to_raw -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode @@ -88,7 +88,7 @@ class Headers(CaselessDict): """Return headers as a CaselessDict with unicode keys and unicode values. Multiple values are joined with ','. """ - return CaselessDict( + return CaseInsensitiveDict( ( to_unicode(key, encoding=self.encoding), to_unicode(b",".join(value), encoding=self.encoding), diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5289f014a..7fc54b5d3 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,10 +4,11 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations import json from contextlib import suppress -from typing import Generator, Tuple +from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple from urllib.parse import urljoin import parsel @@ -25,6 +26,9 @@ from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +if TYPE_CHECKING: + from scrapy.selector import Selector + _NONE = object() @@ -34,11 +38,11 @@ class TextResponse(Response): attributes: Tuple[str, ...] = Response.attributes + ("encoding",) - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): self._encoding = kwargs.pop("encoding", None) - self._cached_benc = None - self._cached_ubody = None - self._cached_selector = None + self._cached_benc: Optional[str] = None + self._cached_ubody: Optional[str] = None + self._cached_selector: Optional[Selector] = None super().__init__(*args, **kwargs) def _set_url(self, url): @@ -82,7 +86,7 @@ class TextResponse(Response): return self._cached_decoded_json @property - def text(self): + def text(self) -> str: """Body as unicode""" # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called diff --git a/scrapy/link.py b/scrapy/link.py index 704649731..0868ae5ef 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ +from typing import Any class Link: @@ -26,16 +27,20 @@ class Link: __slots__ = ["url", "text", "fragment", "nofollow"] - def __init__(self, url, text="", fragment="", nofollow=False): + def __init__( + self, url: str, text: str = "", fragment: str = "", nofollow: bool = False + ): if not isinstance(url, str): got = url.__class__.__name__ raise TypeError(f"Link urls must be str objects, got {got}") - self.url = url - self.text = text - self.fragment = fragment - self.nofollow = nofollow + self.url: str = url + self.text: str = text + self.fragment: str = fragment + self.nofollow: bool = nofollow - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Link): + raise NotImplementedError return ( self.url == other.url and self.text == other.text @@ -43,12 +48,12 @@ class Link: and self.nofollow == other.nofollow ) - def __hash__(self): + def __hash__(self) -> int: return ( hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) ) - def __repr__(self): + def __repr__(self) -> str: return ( f"Link(url={self.url!r}, text={self.text!r}, " f"fragment={self.fragment!r}, nofollow={self.nofollow!r})" diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 7cb379b46..9b05e1153 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging import os -from typing import Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.python.failure import Failure @@ -8,6 +10,13 @@ from scrapy import Request, Spider from scrapy.http import Response from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" @@ -105,7 +114,7 @@ class LogFormatter: } def item_error( - self, item: Any, exception, response: Response, spider: Spider + self, item: Any, exception: BaseException, response: Response, spider: Spider ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -161,5 +170,5 @@ class LogFormatter: } @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls() diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..090588130 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,21 @@ +from __future__ import annotations + import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from twisted.internet.defer import Deferred @@ -11,6 +25,13 @@ from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) @@ -34,7 +55,9 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings(cls, settings: Settings, crawler=None): + def from_settings( + cls, settings: Settings, crawler: Optional[Crawler] = None + ) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -46,10 +69,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) @@ -64,7 +86,7 @@ class MiddlewareManager: return cls(*middlewares) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls.from_settings(crawler.settings, crawler) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b594ccb7..5c09ab37e 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -28,7 +28,7 @@ from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import md5sum @@ -155,7 +155,7 @@ class S3FilesStore: def _headers_to_botocore_kwargs(self, headers): """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. - mapping = CaselessDict( + mapping = CaseInsensitiveDict( { "Content-Type": "ContentType", "Cache-Control": "CacheControl", diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6cbe01cbf..e2e8beff4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,3 +1,5 @@ +from typing import Any + from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( @@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache # TODO: cache misses -dnscache = LocalCache(10000) +dnscache: LocalCache[str, Any] = LocalCache(10000) @implementer(IResolverSimple) @@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver): def install_on_reactor(self): self.reactor.installResolver(self) - def getHostByName(self, name, timeout=None): + def getHostByName(self, name: str, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with @@ -110,7 +112,7 @@ class CachingHostnameResolver: def resolveHostName( self, resolutionReceiver, - hostName, + hostName: str, portNumber=0, addressTypes=None, transportSemantics="TCP", diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 58884f21a..9e411d4aa 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -5,6 +5,7 @@ based on different criteria. from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data +from typing import Dict, Mapping, Optional, Type, Union from scrapy.http import Response from scrapy.utils.misc import load_object @@ -29,15 +30,19 @@ class ResponseTypes: "text/*": "scrapy.http.TextResponse", } - def __init__(self): - self.classes = {} - self.mimetypes = MimeTypes() - mimedata = get_data("scrapy", "mime.types").decode("utf8") - self.mimetypes.readfp(StringIO(mimedata)) + def __init__(self) -> None: + self.classes: Dict[str, Type[Response]] = {} + self.mimetypes: MimeTypes = MimeTypes() + mimedata = get_data("scrapy", "mime.types") + if not mimedata: + raise ValueError( + "The mime.types file is not found in the Scrapy installation" + ) + self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype): + def from_mimetype(self, mimetype: str) -> Type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -46,7 +51,9 @@ class ResponseTypes: basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) - def from_content_type(self, content_type, content_encoding=None): + def from_content_type( + self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + ) -> Type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -56,7 +63,9 @@ class ResponseTypes: ) return self.from_mimetype(mimetype) - def from_content_disposition(self, content_disposition): + def from_content_disposition( + self, content_disposition: Union[str, bytes] + ) -> Type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -68,7 +77,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers): + def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -81,14 +90,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename): + def from_filename(self, filename: str) -> Type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body): + def from_body(self, body: bytes) -> Type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -106,7 +115,13 @@ class ResponseTypes: return self.from_mimetype("text/html") return self.from_mimetype("text") - def from_args(self, headers=None, url=None, filename=None, body=None): + def from_args( + self, + headers: Optional[Mapping[bytes, bytes]] = None, + url: Optional[str] = None, + filename: Optional[str] = None, + body: Optional[bytes] = None, + ) -> Type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index caff79e9c..863fb6032 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,11 @@ """ XPath selectors based on lxml """ +from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response, st): +def _st(response: Optional[TextResponse], st: Optional[str]) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text, st): - rt = XmlResponse if st == "xml" else HtmlResponse +def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: + rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): + def __init__( + self, + response: Optional[TextResponse] = None, + text: Optional[str] = None, + type: Optional[str] = None, + root: Optional[Any] = _NOT_SET, + **kwargs: Any, + ): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index a3b849f7b..ba9727bac 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,21 +1,52 @@ +from __future__ import annotations + import copy import json -from collections.abc import MutableMapping from importlib import import_module from pprint import pformat +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Optional, + Tuple, + Union, + cast, +) from scrapy.settings import default_settings -SETTINGS_PRIORITIES = { +# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, +# see https://github.com/scrapy/scrapy/issues/5383. +_SettingsKeyT = Union[bool, float, int, str, None] + +if TYPE_CHECKING: + # https://github.com/python/typing/issues/445#issuecomment-1131458824 + from _typeshed import SupportsItems + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + + +SETTINGS_PRIORITIES: Dict[str, int] = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, } -def get_settings_priority(priority): +def get_settings_priority(priority: Union[int, str]) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -34,14 +65,15 @@ class SettingsAttribute: for settings configuration, not this one. """ - def __init__(self, value, priority): - self.value = value + def __init__(self, value: Any, priority: int): + self.value: Any = value + self.priority: int if isinstance(self.value, BaseSettings): self.priority = max(self.value.maxpriority(), priority) else: self.priority = priority - def set(self, value, priority): + def set(self, value: Any, priority: int) -> None: """Sets value if priority is higher or equal than current priority.""" if priority >= self.priority: if isinstance(self.value, BaseSettings): @@ -49,11 +81,11 @@ class SettingsAttribute: self.value = value self.priority = priority - def __repr__(self): + def __repr__(self) -> str: return f"" -class BaseSettings(MutableMapping): +class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -75,21 +107,25 @@ class BaseSettings(MutableMapping): highest priority will be retrieved. """ - def __init__(self, values=None, priority="project"): - self.frozen = False - self.attributes = {} + __default = object() + + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): + self.frozen: bool = False + self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name): + def __getitem__(self, opt_name: _SettingsKeyT) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value - def __contains__(self, name): + def __contains__(self, name: Any) -> bool: return name in self.attributes - def get(self, name, default=None): + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -101,7 +137,7 @@ class BaseSettings(MutableMapping): """ return self[name] if self[name] is not None else default - def getbool(self, name, default=False): + def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -131,7 +167,7 @@ class BaseSettings(MutableMapping): "'True'/'False' and 'true'/'false'" ) - def getint(self, name, default=0): + def getint(self, name: _SettingsKeyT, default: int = 0) -> int: """ Get a setting value as an int. @@ -143,7 +179,7 @@ class BaseSettings(MutableMapping): """ return int(self.get(name, default)) - def getfloat(self, name, default=0.0): + def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -155,7 +191,9 @@ class BaseSettings(MutableMapping): """ return float(self.get(name, default)) - def getlist(self, name, default=None): + def getlist( + self, name: _SettingsKeyT, default: Optional[List[Any]] = None + ) -> List[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -174,7 +212,9 @@ class BaseSettings(MutableMapping): value = value.split(",") return list(value) - def getdict(self, name, default=None): + def getdict( + self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None + ) -> Dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -195,7 +235,11 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def getdictorlist(self, name, default=None): + def getdictorlist( + self, + name: _SettingsKeyT, + default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, + ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -222,24 +266,31 @@ class BaseSettings(MutableMapping): return {} if isinstance(value, str): try: - return json.loads(value) + value_loaded = json.loads(value) + assert isinstance(value_loaded, (dict, list)) + return value_loaded except ValueError: return value.split(",") + if isinstance(value, tuple): + return list(value) + assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name): + def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": """Get a composition of a dictionary-like setting and its `_BASE` counterpart. :param name: name of the dictionary-like setting :type name: str """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") compbs = BaseSettings() compbs.update(self[name + "_BASE"]) compbs.update(self[name]) return compbs - def getpriority(self, name): + def getpriority(self, name: _SettingsKeyT) -> Optional[int]: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -251,7 +302,7 @@ class BaseSettings(MutableMapping): return None return self.attributes[name].priority - def maxpriority(self): + def maxpriority(self) -> int: """ Return the numerical value of the highest priority present throughout all settings, or the numerical value for ``default`` from @@ -259,13 +310,15 @@ class BaseSettings(MutableMapping): stored. """ if len(self) > 0: - return max(self.getpriority(name) for name in self) + return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") - def __setitem__(self, name, value): + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) - def set(self, name, value, priority="project"): + def set( + self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + ) -> None: """ Store a key/value attribute with a given priority. @@ -293,17 +346,26 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) - def setdefault(self, name, default=None, priority="project"): + def setdefault( + self, + name: _SettingsKeyT, + default: Any = None, + priority: Union[int, str] = "project", + ) -> Any: if name not in self: self.set(name, default, priority) return default return self.attributes[name].value - def setdict(self, values, priority="project"): + def setdict( + self, values: _SettingsInputT, priority: Union[int, str] = "project" + ) -> None: self.update(values, priority) - def setmodule(self, module, priority="project"): + def setmodule( + self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + ) -> None: """ Store settings from a module with a given priority. @@ -325,7 +387,8 @@ class BaseSettings(MutableMapping): if key.isupper(): self.set(key, getattr(module, key), priority) - def update(self, values, priority="project"): + # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports + def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -349,30 +412,34 @@ class BaseSettings(MutableMapping): """ self._assert_mutability() if isinstance(values, str): - values = json.loads(values) + values = cast(dict, json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, values.getpriority(name)) + self.set(name, value, cast(int, values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) - def delete(self, name, priority="project"): + def delete( + self, name: _SettingsKeyT, priority: Union[int, str] = "project" + ) -> None: + if name not in self: + raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= self.getpriority(name): + if priority >= cast(int, self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name): + def __delitem__(self, name: _SettingsKeyT) -> None: self._assert_mutability() del self.attributes[name] - def _assert_mutability(self): + def _assert_mutability(self) -> None: if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self): + def copy(self) -> "Self": """ Make a deep copy of current settings. @@ -384,7 +451,7 @@ class BaseSettings(MutableMapping): """ return copy.deepcopy(self) - def freeze(self): + def freeze(self) -> None: """ Disable further changes to the current settings. @@ -394,7 +461,7 @@ class BaseSettings(MutableMapping): """ self.frozen = True - def frozencopy(self): + def frozencopy(self) -> "Self": """ Return an immutable copy of the current settings. @@ -404,26 +471,26 @@ class BaseSettings(MutableMapping): copy.freeze() return copy - def __iter__(self): + def __iter__(self) -> Iterator[_SettingsKeyT]: return iter(self.attributes) - def __len__(self): + def __len__(self) -> int: return len(self.attributes) - def _to_dict(self): + def _to_dict(self) -> Dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value): + def _get_key(self, key_value: Any) -> _SettingsKeyT: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self): + def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -439,12 +506,25 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() - def _repr_pretty_(self, p, cycle): + # https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing + def _repr_pretty_(self, p: Any, cycle: bool) -> None: if cycle: p.text(repr(self)) else: p.text(pformat(self.copy_to_dict())) + def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: + try: + value = self.attributes[name].value + except KeyError: + if default is self.__default: + raise + + return default + else: + self.__delitem__(name) + return value + class Settings(BaseSettings): """ @@ -457,7 +537,9 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values=None, priority="project"): + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -471,14 +553,16 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings(): +def iter_default_settings() -> Iterable[Tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): yield name, getattr(default_settings, name) -def overridden_settings(settings): +def overridden_settings( + settings: Mapping[_SettingsKeyT, Any] +) -> Iterable[Tuple[str, Any]]: """Return a dict of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9660e0bcd..b565782f3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,8 @@ import sys from importlib import import_module from pathlib import Path +ADDONS = {} + AJAXCRAWL_ENABLED = False ASYNCIO_EVENT_LOOP = None @@ -141,7 +143,7 @@ EXTENSIONS_BASE = { FEED_TEMPDIR = None FEEDS = {} FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = False +FEED_STORE_EMPTY = True FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None FEED_STORAGES = {} @@ -262,6 +264,21 @@ RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 +RETRY_EXCEPTIONS = [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.DNSLookupError", + "twisted.internet.error.ConnectionRefusedError", + "twisted.internet.error.ConnectionDone", + "twisted.internet.error.ConnectError", + "twisted.internet.error.ConnectionLost", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + # OSError is raised by the HttpCompression middleware when trying to + # decompress an empty response + OSError, + "scrapy.core.downloader.handlers.http11.TunnelError", +] ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 02a451a2b..d855c962c 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,13 +1,23 @@ +from __future__ import annotations + import traceback import warnings from collections import defaultdict +from types import ModuleType +from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer +from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader +from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(ISpiderLoader) class SpiderLoader: @@ -16,14 +26,14 @@ class SpiderLoader: in a Scrapy project. """ - def __init__(self, settings): - self.spider_modules = settings.getlist("SPIDER_MODULES") - self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders = {} - self._found = defaultdict(list) + def __init__(self, settings: BaseSettings): + self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") + self._spiders: Dict[str, Type[Spider]] = {} + self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() - def _check_name_duplicates(self): + def _check_name_duplicates(self) -> None: dupes = [] for name, locations in self._found.items(): dupes.extend( @@ -42,12 +52,12 @@ class SpiderLoader: category=UserWarning, ) - def _load_spiders(self, module): + def _load_spiders(self, module: ModuleType) -> None: for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): + def _load_all_spiders(self) -> None: for name in self.spider_modules: try: for module in walk_modules(name): @@ -65,10 +75,10 @@ class SpiderLoader: self._check_name_duplicates() @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name): + def load(self, spider_name: str) -> Type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -78,7 +88,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request): + def find_by_request(self, request: Request) -> List[str]: """ Return the list of spider names that can handle the given request. """ @@ -86,7 +96,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self): + def list(self) -> List[str]: """ Return a list with the names of all spiders available in the project. """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 3502f8b27..e16d71727 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast + +from twisted.internet.defer import Deferred from scrapy import signals -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class Spider(object_ref): @@ -25,21 +31,21 @@ class Spider(object_ref): name: str custom_settings: Optional[dict] = None - def __init__(self, name=None, **kwargs): + def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: self.name = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls = [] + self.start_urls: List[str] = [] @property - def logger(self): + def logger(self) -> logging.LoggerAdapter: logger = logging.getLogger(self.name) return logging.LoggerAdapter(logger, {"spider": self}) - def log(self, message, level=logging.DEBUG, **kw): + def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level This helper wraps a log call to the logger within the spider, but you @@ -49,17 +55,17 @@ class Spider(object_ref): self.logger.log(level, message, **kw) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = cls(*args, **kwargs) spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler: Crawler): + def _set_crawler(self, crawler: Crawler) -> None: self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self): + def start_requests(self) -> Iterable[Request]: if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " @@ -69,29 +75,30 @@ class Spider(object_ref): for url in self.start_urls: yield Request(url, dont_filter=True) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response, **kwargs): + def parse(self, response: Response, **kwargs: Any) -> Any: raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) @classmethod - def update_settings(cls, settings): + def update_settings(cls, settings: BaseSettings) -> None: settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod - def handles_request(cls, request): + def handles_request(cls, request: Request) -> bool: return url_is_from_spider(request.url, cls) @staticmethod - def close(spider, reason): + def close(spider: Spider, reason: str) -> Union[Deferred, None]: closed = getattr(spider, "closed", None) if callable(closed): - return closed(reason) + return cast(Union[Deferred, None], closed(reason)) + return None - def __repr__(self): + def __repr__(self) -> str: return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6afe0d636..f665ad88c 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -10,7 +10,6 @@ from typing import Union from queuelib import queue -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.request import request_from_dict @@ -143,32 +142,3 @@ MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDisk MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) - - -# deprecated queue classes -_subclass_warn_message = "{cls} inherits from deprecated class {old}" -_instance_warn_message = "{cls} is deprecated" -PickleFifoDiskQueueNonRequest = create_deprecated_class( - name="PickleFifoDiskQueueNonRequest", - new_class=_PickleFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -PickleLifoDiskQueueNonRequest = create_deprecated_class( - name="PickleLifoDiskQueueNonRequest", - new_class=_PickleLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalFifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalFifoDiskQueueNonRequest", - new_class=_MarshalFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalLifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalLifoDiskQueueNonRequest", - new_class=_MarshalLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..53cfeddd0 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,7 @@ """Boto/botocore helpers""" -def is_botocore_available(): +def is_botocore_available() -> bool: try: import botocore # noqa: F401 diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..641dfa4a2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,18 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Union, +) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -13,21 +24,26 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, custom=None, convert=update_classpath): +def build_component_list( + compdict: MutableMapping[Any, Any], + custom: Any = None, + convert: Callable[[Any], Any] = update_classpath, +) -> List[Any]: """Compose a component list from a { class: order } dictionary.""" - def _check_components(complist): + def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): raise ValueError( f"Some paths in {complist!r} convert to the same object, " "please update your settings" ) - def _map_keys(compdict): + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): prio = compdict.getpriority(k) + assert prio is not None if compbs.getpriority(convert(k)) == prio: raise ValueError( f"Some paths in {list(compdict.keys())!r} " @@ -40,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in compdict.items()} - def _validate_values(compdict): + def _validate_values(compdict: Mapping[Any, Any]) -> None: """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): @@ -49,11 +65,17 @@ def build_component_list(compdict, custom=None, convert=update_classpath): "please provide a real number or None instead" ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) - if custom is not None: + warnings.warn( + "The 'custom' attribute of build_component_list() is deprecated. " + "Please merge its value into 'compdict' manually or change your " + "code to use Settings.getwithbase().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) # type: ignore[return-value] compdict.update(custom) _validate_values(compdict) @@ -61,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist): +def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -84,7 +106,7 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def init_env(project="default", set_syspath=True): +def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to be able to locate the project module. @@ -99,7 +121,7 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): +def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) cfg = ConfigParser() @@ -107,7 +129,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> List[str]: +def get_sources(use_closest: bool = True) -> List[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -122,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]: return sources -def feed_complete_default_values_from_settings(feed, settings): +def feed_complete_default_values_from_settings( + feed: Dict[str, Any], settings: BaseSettings +) -> Dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -138,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings): def feed_process_params_from_cli( - settings, + settings: BaseSettings, output: List[str], - output_format=None, + output_format: Optional[str] = None, overwrite_output: Optional[List[str]] = None, -): +) -> Dict[str, Dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ - valid_output_formats = without_none_values( + valid_output_formats: Iterable[str] = without_none_values( settings.getwithbase("FEED_EXPORTERS") ).keys() - def check_valid_format(output_format): + def check_valid_format(output_format: str) -> None: if output_format not in valid_output_formats: raise UsageError( f"Unrecognized output format '{output_format}'. " @@ -202,7 +226,8 @@ def feed_process_params_from_cli( for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) - except ValueError: + check_valid_format(feed_format) + except (ValueError, UsageError): feed_uri = element feed_format = Path(element).suffix.replace(".", "") else: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index fa57a4f26..d5b9544cc 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,13 +6,32 @@ This module must not depend on any module outside the Standard Library. """ import collections +import warnings import weakref from collections.abc import Mapping +from typing import Any, AnyStr, Optional, OrderedDict, Sequence, TypeVar + +from scrapy.exceptions import ScrapyDeprecationWarning + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") class CaselessDict(dict): __slots__ = () + def __new__(cls, *args, **kwargs): + from scrapy.http.headers import Headers + + if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): + warnings.warn( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, *args, **kwargs) + def __init__(self, seq=None): super().__init__() if seq: @@ -64,17 +83,59 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict): +class CaseInsensitiveDict(collections.UserDict): + """A dict-like structure that accepts strings or bytes + as keys and allows case-insensitive lookups. + """ + + def __init__(self, *args, **kwargs) -> None: + self._keys: dict = {} + super().__init__(*args, **kwargs) + + def __getitem__(self, key: AnyStr) -> Any: + normalized_key = self._normkey(key) + return super().__getitem__(self._keys[normalized_key.lower()]) + + def __setitem__(self, key: AnyStr, value: Any) -> None: + normalized_key = self._normkey(key) + try: + lower_key = self._keys[normalized_key.lower()] + del self[lower_key] + except KeyError: + pass + super().__setitem__(normalized_key, self._normvalue(value)) + self._keys[normalized_key.lower()] = normalized_key + + def __delitem__(self, key: AnyStr) -> None: + normalized_key = self._normkey(key) + stored_key = self._keys.pop(normalized_key.lower()) + super().__delitem__(stored_key) + + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + normalized_key = self._normkey(key) + return normalized_key.lower() in self._keys + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {super().__repr__()}>" + + def _normkey(self, key: AnyStr) -> AnyStr: + return key + + def _normvalue(self, value: Any) -> Any: + return value + + +class LocalCache(OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.limit = limit + self.limit: Optional[int] = limit - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: while len(self) >= self.limit: self.popitem(last=False) @@ -93,17 +154,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.data = LocalCache(limit=limit) + self.data: LocalCache = LocalCache(limit=limit) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: try: super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): @@ -113,8 +174,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq): - self.seq = seq + def __init__(self, seq: Sequence): + self.seq: Sequence = seq - def __contains__(self, item): + def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4e684645b..04186559f 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,19 +1,21 @@ import warnings from functools import wraps +from typing import Any, Callable from twisted.internet import defer, threads +from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning -def deprecated(use_instead=None): +def deprecated(use_instead: Any = None) -> Callable: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func): + def deco(func: Callable) -> Callable: @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: Any, **kwargs: Any) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -28,23 +30,23 @@ def deprecated(use_instead=None): return deco -def defers(func): +def defers(func: Callable) -> Callable[..., Deferred]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func): +def inthread(func: Callable) -> Callable[..., Deferred]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..bf3c5ef5b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,19 +5,26 @@ import asyncio import inspect from asyncio import Future from functools import wraps +from types import CoroutineType from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Awaitable, Callable, Coroutine, + Dict, Generator, Iterable, Iterator, List, Optional, + Tuple, + TypeVar, Union, cast, + overload, ) from twisted.internet import defer @@ -44,7 +51,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result) -> Deferred: +def defer_succeed(result: Any) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -58,7 +65,7 @@ def defer_succeed(result) -> Deferred: return d -def defer_result(result) -> Deferred: +def defer_result(result: Any) -> Deferred: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -66,7 +73,7 @@ def defer_result(result) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: +def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -84,7 +91,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args, **named + iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -146,14 +153,14 @@ class _AsyncCooperatorAdapter(Iterator): self, aiterable: AsyncIterable, callable: Callable, - *callable_args, - **callable_kwargs + *callable_args: Any, + **callable_kwargs: Any, ): - self.aiterator = aiterable.__aiter__() - self.callable = callable - self.callable_args = callable_args - self.callable_kwargs = callable_kwargs - self.finished = False + self.aiterator: AsyncIterator = aiterable.__aiter__() + self.callable: Callable = callable + self.callable_args: Tuple[Any, ...] = callable_args + self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] self.anext_deferred: Optional[Deferred] = None @@ -183,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = cast( - Deferred, deferred_from_coro(self.aiterator.__anext__()) - ) + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -201,7 +206,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named + async_iterable: AsyncIterable, + count: int, + callable: Callable, + *args: Any, + **named: Any, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -210,7 +219,9 @@ def parallel_async( return dl -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_chain( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() for x in callbacks: @@ -220,7 +231,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: def process_chain_both( - callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw + callbacks: Iterable[Callable], + errbacks: Iterable[Callable], + input: Any, + *a: Any, + **kw: Any, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() @@ -240,7 +255,9 @@ def process_chain_both( return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_parallel( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -250,7 +267,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: +def iter_errback( + iterable: Iterable, errback: Callable, *a: Any, **kw: Any +) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -265,7 +284,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a, **kw + aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any ) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() @@ -280,7 +299,21 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o) -> Any: +_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +_T = TypeVar("_T") + + +@overload +def deferred_from_coro(o: _CT) -> Deferred: + ... + + +@overload +def deferred_from_coro(o: _T) -> _T: + ... + + +def deferred_from_coro(o: _T) -> Union[Deferred, _T]: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -303,13 +336,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args, **coro_kwargs): + def f(*coro_args: Any, **coro_kwargs: Any) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: +def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) @@ -340,8 +373,9 @@ def deferred_to_future(d: Deferred) -> Future: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - additional_response = await deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await deferred_to_future(deferred) """ return d.asFuture(_get_asyncio_event_loop()) @@ -368,8 +402,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - extra_response = await maybe_deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) """ if not is_asyncio_reactor_installed(): return d diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f4d6e0451..ea577c44a 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,12 +2,12 @@ import inspect import warnings -from typing import List, Tuple +from typing import Any, Dict, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning -def attribute(obj, oldattr, newattr, version="0.12"): +def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: cname = obj.__class__.__name__ warnings.warn( f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " @@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"): def create_deprecated_class( - name, - new_class, - clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", - instance_warn_message="{cls} is deprecated, instantiate {new} instead.", -): + name: str, + new_class: type, + clsdict: Optional[Dict[str, Any]] = None, + warn_category: Type[Warning] = ScrapyDeprecationWarning, + warn_once: bool = True, + old_class_path: Optional[str] = None, + new_class_path: Optional[str] = None, + subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", +) -> type: """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. @@ -53,17 +53,20 @@ def create_deprecated_class( OldName. """ - class DeprecatedClass(new_class.__class__): - deprecated_class = None - warned_on_subclass = False + # https://github.com/python/mypy/issues/4177 + class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + deprecated_class: Optional[type] = None + warned_on_subclass: bool = False - def __new__(metacls, name, bases, clsdict_): + def __new__( + metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name, bases, clsdict_): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -81,10 +84,10 @@ def create_deprecated_class( # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details - def __instancecheck__(cls, inst): + def __instancecheck__(cls, inst: Any) -> bool: return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) - def __subclasscheck__(cls, sub): + def __subclasscheck__(cls, sub: type) -> bool: if cls is not DeprecatedClass.deprecated_class: # we should do the magic only if second `issubclass` argument # is the deprecated class itself - subclasses of the @@ -98,7 +101,7 @@ def create_deprecated_class( mro = getattr(sub, "__mro__", ()) return any(c in {cls, new_class} for c in mro) - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: old = DeprecatedClass.deprecated_class if cls is old: msg = instance_warn_message.format( @@ -125,7 +128,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls, forced=None): +def _clspath(cls: type, forced: Optional[str] = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" @@ -134,7 +137,17 @@ def _clspath(cls, forced=None): DEPRECATION_RULES: List[Tuple[str, str]] = [] -def update_classpath(path): +@overload +def update_classpath(path: str) -> str: + ... + + +@overload +def update_classpath(path: Any) -> Any: + ... + + +def update_classpath(path: Any) -> Any: """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: if isinstance(path, str) and path.startswith(prefix): @@ -147,7 +160,7 @@ def update_classpath(path): return path -def method_is_overridden(subclass, base_class, method_name): +def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool: """ Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 77c32b002..596cf89e4 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,17 +6,18 @@ import ctypes import platform import sys from pprint import pformat as pformat_ +from typing import Any from packaging.version import Version as parse_version -def _enable_windows_terminal_processing(): +def _enable_windows_terminal_processing() -> bool: # https://stackoverflow.com/a/36760881 - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) -def _tty_supports_color(): +def _tty_supports_color() -> bool: if sys.platform != "win32": return True @@ -28,7 +29,7 @@ def _tty_supports_color(): return _enable_windows_terminal_processing() -def _colorize(text, colorize=True): +def _colorize(text: str, colorize: bool = True) -> str: if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: @@ -42,9 +43,9 @@ def _colorize(text, colorize=True): return highlight(text, PythonLexer(), TerminalFormatter()) -def pformat(obj, *args, **kwargs): +def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: return _colorize(pformat_(obj), kwargs.pop("colorize", True)) -def pprint(obj, *args, **kwargs): +def pprint(obj: Any, *args: Any, **kwargs: Any) -> None: print(pformat(obj, *args, **kwargs)) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 8e3ec2c37..a5f2a8c6e 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,9 +2,13 @@ # used in global tests code from time import time # noqa: F401 +from typing import TYPE_CHECKING, Any, List, Tuple + +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine): +def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -23,7 +27,7 @@ def get_engine_status(engine): "engine.scraper.slot.needs_backout()", ] - checks = [] + checks: List[Tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] @@ -33,7 +37,7 @@ def get_engine_status(engine): return checks -def format_engine_status(engine=None): +def format_engine_status(engine: "ExecutionEngine") -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -43,5 +47,5 @@ def format_engine_status(engine=None): return s -def print_engine_status(engine): +def print_engine_status(engine: "ExecutionEngine") -> None: print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6bf6e9195..c77681a53 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,9 +1,10 @@ import posixpath from ftplib import FTP, error_perm from posixpath import dirname +from typing import IO -def ftp_makedirs_cwd(ftp, path, first_call=True): +def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. @@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( - *, path, file, host, port, username, password, use_active_mode=False, overwrite=True -): + *, + path: str, + file: IO, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, + overwrite: bool = True, +) -> None: """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..c7f74030e 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,34 +1,32 @@ import struct from gzip import GzipFile from io import BytesIO +from typing import List + +from scrapy.http import Response -def gunzip(data): +def gunzip(data: bytes) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] + output_list: List[bytes] = [] chunk = b"." while chunk: try: chunk = f.read1(8196) output_list.append(chunk) - except (IOError, EOFError, struct.error): + except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty and f.extrabuf - # contains the whole page content - if output_list or getattr(f, "extrabuf", None): - try: - output_list.append(f.extrabuf[-f.extrasize :]) - finally: - break - else: - raise + # some pages are quite small so output_list is empty + if output_list: + break + raise return b"".join(output_list) -def gzip_magic_number(response): +def gzip_magic_number(response: Response) -> bool: return response.body[:3] == b"\x1f\x8b\x08" diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 170055d5e..03d779afb 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -2,15 +2,34 @@ import csv import logging import re from io import StringIO +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Literal, + Optional, + Union, + cast, + overload, +) from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode +if TYPE_CHECKING: + from lxml._types import SupportsReadClose + logger = logging.getLogger(__name__) -def xmliter(obj, nodename): +def xmliter( + obj: Union[Response, str, bytes], nodename: str +) -> Generator[Selector, Any, None]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -27,20 +46,22 @@ def xmliter(obj, nodename): NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) - document_header = re.search(DOCUMENT_HEADER_RE, text) - document_header = document_header.group().strip() if document_header else "" + document_header_match = re.search(DOCUMENT_HEADER_RE, text) + document_header = ( + document_header_match.group().strip() if document_header_match else "" + ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces = {} + namespaces: Dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): + assert header_end_idx tag = re.search( rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: - namespaces.update( - reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()) - ) + for x in re.findall(NAMESPACE_RE, tag.group()): + namespaces[x[1]] = x[0] r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): @@ -54,12 +75,19 @@ def xmliter(obj, nodename): yield Selector(text=nodetext, type="xml") -def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): +def xmliter_lxml( + obj: Union[Response, str, bytes], + nodename: str, + namespace: Optional[str] = None, + prefix: str = "x", +) -> Generator[Selector, Any, None]: from lxml import etree reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) + iterable = etree.iterparse( + cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding="unicode") @@ -71,30 +99,46 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): class _StreamReader: - def __init__(self, obj): - self._ptr = 0 - if isinstance(obj, Response): + def __init__(self, obj: Union[Response, str, bytes]): + self._ptr: int = 0 + self._text: Union[str, bytes] + if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding + elif isinstance(obj, Response): + self._text, self.encoding = obj.body, "utf-8" else: self._text, self.encoding = obj, "utf-8" - self._is_unicode = isinstance(self._text, str) + self._is_unicode: bool = isinstance(self._text, str) + self._is_first_read: bool = True - def read(self, n=65535): - self.read = self._read_unicode if self._is_unicode else self._read_string - return self.read(n).lstrip() + def read(self, n: int = 65535) -> bytes: + method: Callable[[int], bytes] = ( + self._read_unicode if self._is_unicode else self._read_string + ) + result = method(n) + if self._is_first_read: + self._is_first_read = False + result = result.lstrip() + return result - def _read_string(self, n=65535): + def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e] + return cast(bytes, self._text)[s:e] - def _read_unicode(self, n=65535): + def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e].encode("utf-8") + return cast(str, self._text)[s:e].encode("utf-8") -def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): +def csviter( + obj: Union[Response, str, bytes], + delimiter: Optional[str] = None, + headers: Optional[List[str]] = None, + encoding: Optional[str] = None, + quotechar: Optional[str] = None, +) -> Generator[Dict[str, str], Any, None]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -112,12 +156,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - def row_to_unicode(row_): + def row_to_unicode(row_: Iterable) -> List[str]: return [to_unicode(field, encoding) for field in row_] lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs = {} + kwargs: Dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: @@ -147,7 +191,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): yield dict(zip(headers, row)) -def _body_or_str(obj, unicode=True): +@overload +def _body_or_str(obj: Union[Response, str, bytes]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: + ... + + +def _body_or_str( + obj: Union[Response, str, bytes], unicode: bool = True +) -> Union[str, bytes]: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) @@ -156,10 +217,10 @@ def _body_or_str(obj, unicode=True): ) if isinstance(obj, Response): if not unicode: - return obj.body + return cast(bytes, obj.body) if isinstance(obj, TextResponse): return obj.text - return obj.body.decode("utf-8") + return cast(bytes, obj.body).decode("utf-8") if isinstance(obj, str): return obj if unicode else obj.encode("utf-8") return obj.decode("utf-8") if unicode else obj diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 858affc03..c49f7d758 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path = settings["JOBDIR"] + path: str = settings["JOBDIR"] if path and not Path(path).exists(): Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2ce4725f4..0d17f6153 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import logging import sys import warnings from logging.config import dictConfig -from typing import Tuple +from types import TracebackType +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -12,13 +15,25 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def failure_to_exc_info(failure: Failure): +def failure_to_exc_info( + failure: Failure, +) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): - return (failure.type, failure.value, failure.getTracebackObject()) + assert failure.type + assert failure.value + return ( + failure.type, + failure.value, + cast(Optional[TracebackType], failure.getTracebackObject()), + ) + return None class TopLevelFormatter(logging.Filter): @@ -33,10 +48,10 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers=None): - self.loggers = loggers or [] + def __init__(self, loggers: Optional[List[str]] = None): + self.loggers: List[str] = loggers or [] - def filter(self, record): + def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): record.name = record.name.split(".", 1)[0] return True @@ -62,7 +77,9 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None, install_root_handler=True): +def configure_logging( + settings: Union[Settings, dict, None] = None, install_root_handler: bool = True +) -> None: """ Initialize logging defaults for Scrapy. @@ -99,13 +116,16 @@ def configure_logging(settings=None, install_root_handler=True): settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) + sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] if install_root_handler: install_scrapy_root_handler(settings) -def install_scrapy_root_handler(settings): +_scrapy_root_handler: Optional[logging.Handler] = None + + +def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler if ( @@ -118,16 +138,14 @@ def install_scrapy_root_handler(settings): logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler(): +def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler = None - - -def _get_handler(settings): +def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") + handler: logging.Handler if filename: mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w" encoding = settings.get("LOG_ENCODING") @@ -181,16 +199,16 @@ class StreamLogger: https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ - def __init__(self, logger, log_level=logging.INFO): - self.logger = logger - self.log_level = log_level - self.linebuf = "" + def __init__(self, logger: logging.Logger, log_level: int = logging.INFO): + self.logger: logging.Logger = logger + self.log_level: int = log_level + self.linebuf: str = "" - def write(self, buf): + def write(self, buf: str) -> None: for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) - def flush(self): + def flush(self) -> None: for h in self.logger.handlers: h.flush() @@ -198,11 +216,11 @@ class StreamLogger: class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" - def __init__(self, crawler, *args, **kwargs): + def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.crawler = crawler + self.crawler: Crawler = crawler - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..b3c28da92 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,7 +10,21 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import TYPE_CHECKING, Any, Callable, Union +from types import ModuleType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Deque, + Generator, + Iterable, + List, + Optional, + Pattern, + Union, + cast, +) from w3lib.html import replace_entities @@ -26,7 +40,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -def arg_to_iter(arg): +def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -35,7 +49,7 @@ def arg_to_iter(arg): if arg is None: return [] if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return arg + return cast(Iterable[Any], arg) return [arg] @@ -53,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: @@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def walk_modules(path): +def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -80,7 +94,7 @@ def walk_modules(path): For example: walk_modules('scrapy.utils') """ - mods = [] + mods: List[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -94,7 +108,9 @@ def walk_modules(path): return mods -def extract_regex(regex, text, encoding="utf-8"): +def extract_regex( + regex: Union[str, Pattern], text: str, encoding: str = "utf-8" +) -> List[str]: """Extract a list of unicode strings from the given text/encoding using the following policies: * if the regex contains a named group called "extract" that will be returned @@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group("extract")] # named group + # named group + strings = [regex.search(text).group("extract")] # type: ignore[union-attr] except Exception: - strings = regex.findall(text) # full regex or numbered groups + # full regex or numbered groups + strings = regex.findall(text) strings = flatten(strings) if isinstance(text, str): @@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"): ] -def md5sum(file): +def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -140,7 +158,7 @@ def md5sum(file): return m.hexdigest() -def rel_has_nofollow(rel): +def rel_has_nofollow(rel: Optional[str]) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() @@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): @contextmanager -def set_environ(**kwargs): +def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -198,11 +216,11 @@ def set_environ(**kwargs): os.environ[k] = v -def walk_callable(node): +def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo = deque([node]) + todo: Deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() @@ -217,15 +235,15 @@ def walk_callable(node): _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable): +def is_generator_with_return_value(callable: Callable) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise """ if callable in _generator_callbacks_cache: - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node): + def returns_none(return_node: ast.Return) -> bool: value = return_node.value return ( value is None or isinstance(value, ast.NameConstant) and value.value is None @@ -248,10 +266,10 @@ def is_generator_with_return_value(callable): for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) _generator_callbacks_cache[callable] = False - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 7646264a8..2334ea792 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,6 +1,13 @@ import signal +from types import FrameType +from typing import Any, Callable, Dict, Optional, Union -signal_names = {} +# copy of _HANDLER from typeshed/stdlib/signal.pyi +SignalHandlerT = Union[ + Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None +] + +signal_names: Dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) @@ -8,7 +15,9 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function, override_sigint=True): +def install_shutdown_handlers( + function: SignalHandlerT, override_sigint: bool = True +) -> None: """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the SIGINT handler won't be install if there is already a handler in place diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index ab1b8e3ee..a2c224b90 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -11,9 +11,9 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" -def inside_project(): - scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE") - if scrapy_module is not None: +def inside_project() -> bool: + scrapy_module = os.environ.get(ENVVAR) + if scrapy_module: try: import_module(scrapy_module) except ImportError as exc: @@ -25,7 +25,7 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project="default") -> str: +def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") @@ -44,7 +44,7 @@ def project_data_dir(project="default") -> str: return str(d) -def data_path(path: str, createdir=False) -> str: +def data_path(path: str, createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. @@ -60,7 +60,7 @@ def data_path(path: str, createdir=False) -> str: return str(path_obj) -def get_project_settings(): +def get_project_settings() -> Settings: if ENVVAR not in os.environ: project = os.environ.get("SCRAPY_PROJECT", "default") init_env(project) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..0b5dc324f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import collections.abc import gc import inspect import re @@ -12,9 +13,17 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Callable, + Dict, + Generator, Iterable, + Iterator, + List, Mapping, Optional, + Pattern, + Tuple, Union, overload, ) @@ -22,7 +31,7 @@ from typing import ( from scrapy.utils.asyncgen import as_async_generator -def flatten(x): +def flatten(x: Iterable) -> list: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -42,7 +51,7 @@ def flatten(x): return list(iflatten(x)) -def iflatten(x): +def iflatten(x: Iterable) -> Iterable: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" @@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_, key=lambda x: x): +def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: """efficient function to uniquify a list preserving item order""" seen = set() result = [] @@ -124,7 +133,9 @@ def to_bytes( return text.encode(encoding, errors) -def re_rsearch(pattern, text, chunk_size=1024): +def re_rsearch( + pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 +) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024): the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter(): + def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024): return None -def memoizemethod_noargs(method): +def memoizemethod_noargs(method: Callable) -> Callable: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self, *args, **kwargs): + def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func, stripself=False): +def get_func_args(func: Callable, stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args = [] + args: List[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -217,7 +228,7 @@ def get_func_args(func, stripself=False): return args -def get_spec(func): +def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -246,7 +257,7 @@ def get_spec(func): else: raise TypeError(f"{type(func)} is not callable") - defaults = spec.defaults or [] + defaults: Tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -254,7 +265,9 @@ def get_spec(func): return args, kwargs -def equal_attributes(obj1, obj2, attributes): +def equal_attributes( + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] +) -> bool: """Compare two objects attributes""" # not attributes given return False by default if not attributes: @@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable: ... -def without_none_values(iterable): +def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - try: + if isinstance(iterable, collections.abc.Mapping): return {k: v for k, v in iterable.items() if v is not None} - except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + else: + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] -def global_object_name(obj): +def global_object_name(obj: Any) -> str: """ Return full name of a global object. @@ -307,14 +321,14 @@ def global_object_name(obj): if hasattr(sys, "pypy_version_info"): - def garbage_collect(): + def garbage_collect() -> None: # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() else: - def garbage_collect(): + def garbage_collect() -> None: gc.collect() @@ -329,10 +343,10 @@ class MutableChain(Iterable): def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self): + def __iter__(self) -> Iterator: return self - def __next__(self): + def __next__(self) -> Any: return next(self.data) @@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable): def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self): + def __aiter__(self) -> AsyncIterator: return self - async def __anext__(self): + async def __anext__(self) -> Any: return await self.data.__anext__() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f1b9239e6..ad3d1d8bc 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,7 +1,8 @@ import asyncio import sys +from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence +from typing import Any, Callable, Dict, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -57,7 +58,7 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def set_asyncio_event_loop_policy(): +def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. This should only be used to install the reactor. @@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy(): _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy(): +def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: warn( "Call to deprecated function " "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" @@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy(): return _get_asyncio_event_loop_policy() -def _get_asyncio_event_loop_policy(): +def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy(): return policy -def install_reactor(reactor_path, event_loop_path=None): +def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None): installer() -def _get_asyncio_event_loop(): +def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path): +def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) + event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: @@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path): return event_loop -def verify_installed_reactor(reactor_path): +def verify_installed_reactor(reactor_path: str) -> None: """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import path.""" @@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) -def verify_installed_asyncio_event_loop(loop_path): +def verify_installed_asyncio_event_loop(loop_path: str) -> None: from twisted.internet import reactor loop_class = load_object(loop_path) @@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path): ) -def is_asyncio_reactor_installed(): +def is_asyncio_reactor_installed() -> bool: from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py deleted file mode 100644 index 15705db83..000000000 --- a/scrapy/utils/reqser.py +++ /dev/null @@ -1,27 +0,0 @@ -import warnings -from typing import Optional - -import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.request import request_from_dict as _from_dict - -warnings.warn( - ( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead" - ), - category=ScrapyDeprecationWarning, - stacklevel=2, -) - - -def request_to_dict( - request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None -) -> dict: - return request.to_dict(spider=spider) - - -def request_from_dict( - d: dict, spider: Optional["scrapy.Spider"] = None -) -> "scrapy.Request": - return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6d8be991d..24fcbd85e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -6,7 +6,19 @@ scrapy.http.Request objects import hashlib import json import warnings -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + Union, +) from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -19,11 +31,16 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy.crawler import Crawler + _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() -def _serialize_headers(headers, request): +def _serialize_headers( + headers: Iterable[bytes], request: Request +) -> Generator[bytes, Any, None]: for header in headers: if header in request.headers: yield header @@ -139,7 +156,7 @@ def request_fingerprint( return cache[cache_key] -def _request_fingerprint_as_bytes(*args, **kwargs): +def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: with warnings.catch_warnings(): warnings.simplefilter("ignore") return bytes.fromhex(request_fingerprint(*args, **kwargs)) @@ -214,6 +231,11 @@ def fingerprint( return cache[cache_key] +class RequestFingerprinterProtocol(Protocol): + def fingerprint(self, request: Request) -> bytes: + ... + + class RequestFingerprinter: """Default fingerprinter. @@ -231,7 +253,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler): return cls(crawler) - def __init__(self, crawler=None): + def __init__(self, crawler: Optional["Crawler"] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -265,7 +287,7 @@ class RequestFingerprinter: f"and '2.7'." ) - def fingerprint(self, request: Request): + def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) @@ -311,7 +333,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls = load_object(d["_class"]) if "_class" in d else Request + request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) @@ -320,7 +342,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: return request_cls(**kwargs) -def _get_method(obj, name): +def _get_method(obj: Any, name: Any) -> Any: """Helper function for request_from_dict""" name = str(name) try: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 794678c48..c540d6278 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -42,8 +42,7 @@ def get_meta_refresh( """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 - _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] + _metaref_cache[response] = html.get_meta_refresh( text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 414658944..3b4f67f00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,6 +1,7 @@ import datetime import decimal import json +from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer @@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" - def default(self, o): + def default(self, o: Any) -> Any: if isinstance(o, set): return list(o) if isinstance(o, datetime.datetime): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 9e7ddd827..21a12a19e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) def send_catch_log( - signal=Any, sender=Anonymous, *arguments, **named + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -65,13 +68,18 @@ def send_catch_log( return responses -def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log_deferred( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny +) -> Deferred: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. """ - def logerror(failure, recv): + def logerror(failure: Failure, recv: Any) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): return d -def disconnect_all(signal=Any, sender=Any): +def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running tests """ diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2622c2775..3d2ecc9a7 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ - +from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin import lxml.etree @@ -14,7 +14,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext): + def __init__(self, xmltext: str): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -22,9 +22,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self): + def __iter__(self) -> Iterator[Dict[str, Any]]: for elem in self._root.getchildren(): - d = {} + d: Dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag @@ -39,11 +39,13 @@ class Sitemap: yield d -def sitemap_urls_from_robots(robots_text, base_url=None): +def sitemap_urls_from_robots( + robots_text: str, base_url: Optional[str] = None +) -> Generator[str, Any, None]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith("sitemap:"): url = line.split(":", 1)[1].strip() - yield urljoin(base_url, url) + yield urljoin(base_url or "", url) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 86449eeb2..704df8657 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,14 +1,54 @@ +from __future__ import annotations + import inspect import logging +from types import CoroutineType, ModuleType +from typing import ( + TYPE_CHECKING, + Any, + AsyncGenerator, + Generator, + Iterable, + Literal, + Optional, + Type, + TypeVar, + Union, + overload, +) +from twisted.internet.defer import Deferred + +from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) +_T = TypeVar("_T") -def iterate_spider_output(result): + +# https://stackoverflow.com/questions/60222982 +@overload +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] + ... + + +@overload +def iterate_spider_output(result: CoroutineType) -> Deferred: + ... + + +@overload +def iterate_spider_output(result: _T) -> Iterable: + ... + + +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -18,7 +58,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module): +def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -36,9 +76,46 @@ def iter_spider_classes(module): yield obj +@overload def spidercls_for_request( - spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False -): + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Type[Spider], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Type[Spider]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Literal[None], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + *, + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Optional[Type[Spider]] = None, + log_none: bool = False, + log_multiple: bool = False, +) -> Optional[Type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 03ae4ba9e..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, cast +from typing import Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL @@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]: def get_openssl_version() -> str: - # https://github.com/python/typeshed/issues/10024 - system_openssl_bytes = cast( - bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - ) + system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1499aeb3d..6b22f3bfa 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -4,10 +4,10 @@ import re import string from os import PathLike from pathlib import Path -from typing import Union +from typing import Any, Union -def render_templatefile(path: Union[str, PathLike], **kwargs): +def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") @@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]") -def string_camelcase(string): +def string_camelcase(string: str) -> str: """Convert a word to its CamelCase version and remove invalid chars >>> string_camelcase('lost-pound') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 58576903a..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,24 +7,30 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from unittest import mock +from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from unittest import TestCase, mock +from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available -def assert_gcs_environ(): +def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") -def skip_if_no_boto(): +def skip_if_no_boto() -> None: if not is_botocore_available(): raise SkipTest("missing botocore library") -def get_gcs_content_and_delete(bucket, path): +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> Tuple[bytes, List[Dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path): def get_ftp_content_and_delete( - path, host, port, username, password, use_active_mode=False -): + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: from ftplib import FTP ftp = FTP() @@ -46,19 +57,23 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data = [] + ftp_data: List[bytes] = [] - def buffer_data(data): + def buffer_data(data: bytes) -> None: ftp_data.append(data) ftp.retrbinary(f"RETR {path}", buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) - return "".join(ftp_data) + return b"".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): +def get_crawler( + spidercls: Optional[Type[Spider]] = None, + settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, +) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -82,7 +97,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv(): +def get_testenv() -> Dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -91,21 +106,23 @@ def get_testenv(): return env -def assert_samelines(testcase, text1, text2, msg=None): +def assert_samelines( + testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None +) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value): - q = asyncio.Queue() +def get_from_asyncio_queue(value: Any) -> Coroutine: + q: asyncio.Queue = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter -def mock_google_cloud_storage(): +def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ @@ -122,7 +139,7 @@ def mock_google_cloud_storage(): return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url): +def get_web_client_agent_req(url: str) -> Deferred: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f9bdef37..5f7a7db14 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,7 +1,13 @@ +from __future__ import annotations + import os import sys +from typing import Iterable, Optional, Tuple, cast -from twisted.internet import defer, protocol +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol +from twisted.python.failure import Failure class ProcessTest: @@ -9,7 +15,12 @@ class ProcessTest: prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir - def execute(self, args, check_code=True, settings=None): + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: Optional[str] = None, + ) -> Deferred: from twisted.internet import reactor env = os.environ.copy() @@ -21,29 +32,31 @@ class ProcessTest: reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred - def _process_finished(self, pp, cmd, check_code): + def _process_finished( + self, pp: TestProcessProtocol, cmd: str, check_code: bool + ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err}" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" raise RuntimeError(msg) - return pp.exitcode, pp.out, pp.err + return cast(int, pp.exitcode), pp.out, pp.err -class TestProcessProtocol(protocol.ProcessProtocol): - def __init__(self): - self.deferred = defer.Deferred() - self.out = b"" - self.err = b"" - self.exitcode = None +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: Optional[int] = None - def outReceived(self, data): + def outReceived(self, data: bytes) -> None: self.out += data - def errReceived(self, data): + def errReceived(self, data: bytes) -> None: self.err += data - def processEnded(self, status): - self.exitcode = status.value.exitCode + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode self.deferred.callback(self) diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 01b980c93..9ff9a273f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -12,9 +12,14 @@ alias to object in that case). from collections import defaultdict from operator import itemgetter from time import time -from typing import DefaultDict +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable from weakref import WeakKeyDictionary +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) @@ -24,13 +29,14 @@ class object_ref: __slots__ = () - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> "Self": obj = object.__new__(cls) live_refs[cls][obj] = time() return obj -def format_live_refs(ignore=NoneType): +# using Any as it's hard to type type(None) +def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() @@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType): return s -def print_live_refs(*a, **kw): +def print_live_refs(*a: Any, **kw: Any) -> None: """Print tracked objects""" print(format_live_refs(*a, **kw)) -def get_oldest(class_name): +def get_oldest(class_name: str) -> Any: """Get the oldest object for a specific class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: @@ -58,8 +64,9 @@ def get_oldest(class_name): return min(wdict.items(), key=itemgetter(1))[0] -def iter_all(class_name): +def iter_all(class_name: str) -> Iterable[Any]: """Iterate over all objects of the same class by its class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() + return [] diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 833aa3e20..22b4197f9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import re +from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from scrapy import Spider -def url_is_from_any_domain(url, domains): + +UrlT = Union[str, bytes, ParseResult] + + +def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc.lower() if not host: @@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains): return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url, spider): +def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) ) -def url_has_any_extension(url, extensions): +def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url, encoding=None): +def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ if isinstance(url, ParseResult): return url - return urlparse(to_unicode(url, encoding)) + return cast(ParseResult, urlparse(to_unicode(url, encoding))) -def escape_ajax(url): +def escape_ajax(url: str) -> str: """ Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started @@ -76,7 +83,7 @@ def escape_ajax(url): return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) -def add_http_if_no_scheme(url): +def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) if not match: @@ -87,7 +94,7 @@ def add_http_if_no_scheme(url): return url -def _is_posix_path(string): +def _is_posix_path(string: str) -> bool: return bool( re.match( r""" @@ -109,7 +116,7 @@ def _is_posix_path(string): ) -def _is_windows_path(string): +def _is_windows_path(string: str) -> bool: return bool( re.match( r""" @@ -125,11 +132,11 @@ def _is_windows_path(string): ) -def _is_filesystem_path(string): +def _is_filesystem_path(string: str) -> bool: return _is_posix_path(string) or _is_windows_path(string) -def guess_scheme(url): +def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): @@ -138,12 +145,12 @@ def guess_scheme(url): def strip_url( - url, - strip_credentials=True, - strip_default_port=True, - origin_only=False, - strip_fragment=True, -): + url: str, + strip_credentials: bool = True, + strip_default_port: bool = True, + origin_only: bool = False, + strip_fragment: bool = True, +) -> str: """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index b0737d3d5..9b637bdb0 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,5 +1,6 @@ import platform import sys +from typing import List, Tuple import cryptography import cssselect @@ -12,7 +13,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions(): +def scrapy_components_versions() -> List[Tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index e8affa943..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,113 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) - -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -Managing add-ons -================ - -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. - -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: - - [addons] - httpcache = - -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: - - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py - - -Writing add-ons -=============== - -Add-ons are Python modules that implement the following callbacks. - -addon_configure ---------------- - -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. - -Examples: - -.. code-block:: python - - def addon_configure(settings): - settings.overrides["DOWNLOADER_MIDDLEWARES"].update( - { - "scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware": 900, - } - ) - -.. code-block:: python - - def addon_configure(settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - - -crawler_ready -------------- - -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. - -Examples: - -.. code-block:: python - - def crawler_ready(crawler): - if "some.other.addon" not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") diff --git a/setup.py b/setup.py index c6bcf2439..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,13 @@ from pathlib import Path -from pkg_resources import parse_version -from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() -def has_environment_marker_platform_impl_support(): - """Code extracted from 'pytest/setup.py' - https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 - - The first known release to support environment marker with range operators - it is 18.5, see: - https://setuptools.readthedocs.io/en/latest/history.html#id235 - """ - return parse_version(setuptools_version) >= parse_version("18.5") - - install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.4.6", + "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", @@ -34,21 +21,12 @@ install_requires = [ "setuptools", "packaging", "tldextract", - "lxml>=4.3.0", + "lxml>=4.4.1", ] -extras_require = {} -cpython_dependencies = [ - "PyDispatcher>=2.0.5", -] -if has_environment_marker_platform_impl_support(): - extras_require[ - ':platform_python_implementation == "CPython"' - ] = cpython_dependencies - extras_require[':platform_python_implementation == "PyPy"'] = [ - "PyPyDispatcher>=2.1.0", - ] -else: - install_requires.extend(cpython_dependencies) +extras_require = { + ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], + ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], +} setup( @@ -80,18 +58,18 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 5cc65a903..9b73ca4f0 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from cryptography.hazmat.backends import default_backend @@ -50,8 +50,8 @@ def generate_keys(): .issuer_name(issuer) .public_key(key.public_key()) .serial_number(random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .not_valid_before(datetime.now(tz=timezone.utc)) + .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10)) .add_extension( SubjectAlternativeName([DNSName("localhost")]), critical=False, diff --git a/tests/mockserver.py b/tests/mockserver.py index eb4f8334e..647b0682e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,11 +1,13 @@ import argparse import json +import os import random import sys from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import Dict from urllib.parse import urlencode from OpenSSL import SSL @@ -20,7 +22,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None): return default +def get_mockserver_env() -> Dict[str, str]: + """Return a OS environment dict suitable to run mockserver processes.""" + + tests_path = Path(__file__).parent.parent + pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") + env = os.environ.copy() + env["PYTHONPATH"] = pythonpath + return env + + # most of the following resources are copied from twisted.web.test.test_webclient class ForeverTakingResource(resource.Resource): """ @@ -264,7 +275,7 @@ class MockServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) http_address = self.proc.stdout.readline().strip().decode("ascii") https_address = self.proc.stdout.readline().strip().decode("ascii") @@ -308,7 +319,7 @@ class MockDNSServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) self.host = "127.0.0.1" self.port = int( @@ -331,7 +342,7 @@ class MockFTPServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)], stderr=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) for line in self.proc.stderr: if b"starting FTP server" in line: diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..37186f3a7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,15 +1,17 @@ # Tests requirements attrs -pyftpdlib +# https://github.com/giampaolo/pyftpdlib/issues/560 +pyftpdlib; python_version < "3.12" pytest pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +# uvloop currently doesn't build on 3.12 +uvloop; platform_system != "Windows" and python_version < "3.12" -# optional for shell wrapper tests -bpython +# bpython requires greenlet which currently doesn't build on 3.12 +bpython; python_version < "3.12" # optional for shell wrapper tests brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython diff --git a/tests/spiders.py b/tests/spiders.py index 6ff48f471..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -77,6 +77,22 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class SlowSpider(DelaySpider): + name = "slow" + + def start_requests(self): + # 1st response is fast + url = self.mockserver.url("/delay?n=0&b=0") + yield Request(url, callback=self.parse, errback=self.errback) + + # 2nd response is slow + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") + yield Request(url, callback=self.parse, errback=self.errback) + + def parse(self, response): + yield Item() + + class SimpleSpider(MetaSpider): name = "simple" diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..5d053ed52 --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,158 @@ +import itertools +import unittest +from typing import Any, Dict + +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured +from scrapy.settings import BaseSettings, Settings +from scrapy.utils.test import get_crawler + + +class SimpleAddon: + def update_settings(self, settings): + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig + + +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + +class AddonTest(unittest.TestCase): + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = get_addon_cls(addon_config)() + testaddon.update_settings(settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def test_load_settings(self): + settings_dict = { + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9b39187d5..38ede70e4 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider class TestCloseSpider(TestCase): @@ -54,3 +54,13 @@ class TestCloseSpider(TestCase): self.assertEqual(reason, "closespider_timeout") total_seconds = crawler.stats.get_value("elapsed_time_seconds") self.assertTrue(total_seconds >= close_on) + + @defer.inlineCallbacks + def test_closespider_timeout_no_item(self): + timeout = 1 + crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_timeout_no_item") + total_seconds = crawler.stats.get_value("elapsed_time_seconds") + self.assertTrue(total_seconds >= timeout) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 15833cd19..25ded143c 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,4 +1,5 @@ import json +import os import pstats import shutil import sys @@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv class CmdlineTest(unittest.TestCase): def setUp(self): self.env = get_testenv() + tests_path = Path(__file__).parent.parent + self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): diff --git a/tests/test_commands.py b/tests/test_commands.py index 014f50e92..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -802,17 +800,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, @@ -919,6 +907,64 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", + log, + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", + log, + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", + log, + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", + log, + ) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 813927fc5..1459e0b5f 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -63,6 +63,13 @@ class TestSpider(Spider): """ return Request("http://scrapy.org", callback=self.returns_item) + async def returns_request_async(self, response): + """async method which returns request + @url http://scrapy.org + @returns requests 1 + """ + return Request("http://scrapy.org", callback=self.returns_item) + def returns_item(self, response): """method which returns item @url http://scrapy.org @@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response) self.should_fail() + def test_returns_async(self): + spider = TestSpider() + response = ResponseMock() + + request = self.conman.from_method(spider.returns_request_async, self.results) + request.callback(response) + self.should_error() + def test_scrapes(self): spider = TestSpider() response = ResponseMock() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 706bfbaa9..4c5c48e6d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,15 +1,14 @@ import logging +import os import platform import subprocess import sys import warnings from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from pytest import mark, raises -from twisted import version as twisted_version from twisted.internet import defer -from twisted.python.versions import Version from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -21,10 +20,9 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler, get_testenv -from tests.mockserver import MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer, get_mockserver_env class BaseCrawlerTest(unittest.TestCase): @@ -183,16 +181,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - runner = CrawlerRunner(Settings()) - spiders = runner.spiders - self.assertEqual(len(w), 1) - self.assertIn("CrawlerRunner.spiders", str(w[0].message)) - self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) - sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"]) - self.assertIsInstance(spiders, sl_cls) - class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): @@ -229,14 +217,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -249,7 +237,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: self.fail("Exception should be raised from spider") - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -264,7 +252,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): @@ -289,12 +277,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path + cwd = os.getcwd() def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) p = subprocess.Popen( - args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + args, + env=get_mockserver_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() return stderr.decode("utf-8") @@ -461,17 +453,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -480,17 +462,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -500,17 +472,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -519,17 +481,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fd4176e2f..8459408ff 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase): def test_non_existent(self): request = Request(f"file://{self.mktemp()}") d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, IOError) + return self.assertFailure(d, OSError) class ContentLengthHeaderResource(resource.Resource): @@ -748,7 +748,7 @@ class Http11MockServerTestCase(unittest.TestCase): yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) + self.assertIsNone(failure) reason = crawler.spider.meta["close_reason"] self.assertTrue(reason, "finished") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 060cfe08b..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase): In particular when some website returns a 30x response with header 'Content-Encoding: gzip' giving as result the error below: - exceptions.IOError: Not a gzipped file + BadGzipFile: Not a gzipped file (...) """ req = Request("http://example.com") @@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(IOError, self._download, request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fac5588ff..9dad056de 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -229,7 +229,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_force_recalculate_encoding(self): headers = { @@ -254,7 +254,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): headers = { @@ -277,7 +277,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 63bd61848..97ae1e29a 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,6 @@ import logging import unittest +import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase): == 2 ) - def _test_retry_exception(self, req, exception): + def test_exception_to_retry_added(self): + exc = ValueError + settings_dict = { + "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + } + crawler = get_crawler(Spider, settings_dict=settings_dict) + mw = RetryMiddleware.from_crawler(crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + self._test_retry_exception(req, exc("foo"), mw) + + def test_exception_to_retry_customMiddleware(self): + exc = ValueError + + with warnings.catch_warnings(record=True) as warns: + + class MyRetryMiddleware(RetryMiddleware): + EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) + + self.assertEqual(len(warns), 1) + + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + + def _test_retry_exception(self, req, exception, mw=None): + if mw is None: + mw = self.mw + # first retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 1) # second retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 2) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) diff --git a/tests/test_engine.py b/tests/test_engine.py index 410eba921..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -20,7 +20,6 @@ from threading import Timer from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -29,7 +28,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor @@ -436,90 +435,6 @@ class EngineTest(unittest.TestCase): finally: yield e.stop() - @defer.inlineCallbacks - def test_close_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_crawl_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.crawl(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_download_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.download(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_schedule(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.schedule(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_has_capacity(self): - with pytest.warns( - ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated" - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - self.assertTrue(e.has_capacity()) - spider = TestSpider() - yield e.open_spider(spider, []) - self.assertFalse(e.has_capacity()) - e.start() - yield e.close() - self.assertTrue(e.has_capacity()) - def test_short_timeout(self): args = ( sys.executable, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1491e788e..f4e82705a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,12 +7,10 @@ import tempfile import unittest from datetime import datetime from io import BytesIO -from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, CsvItemExporter, @@ -90,6 +88,10 @@ class BaseItemExporterTest(unittest.TestCase): if self.ie.__class__ is not BaseItemExporter: raise self.ie.finish_exporting() + # Delete the item exporter object, so that if it causes the output + # file handle to be closed, which should not be the case, follow-up + # interactions with the output file handle will surface the issue. + del self.ie self._check_output() def test_export_item(self): @@ -139,7 +141,7 @@ class BaseItemExporterDataclassTest(BaseItemExporterTest): class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(binary=False, **kwargs) + return PythonItemExporter(**kwargs) def test_invalid_option(self): with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): @@ -194,14 +196,6 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported["age"][0]), dict) self.assertEqual(type(exported["age"][0]["age"][0]), dict) - def test_export_binary(self): - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - exporter = PythonItemExporter(binary=True) - value = self.item_class(name="John\xa3", age="22") - expected = {b"name": b"John\xc2\xa3", b"age": b"22"} - self.assertEqual(expected, exporter.export_item(value)) - def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() @@ -243,6 +237,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. f.seek(0) self.assertEqual(self.item_class(**pickle.load(f)), i1) self.assertEqual(self.item_class(**pickle.load(f)), i2) @@ -254,6 +249,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertEqual(pickle.loads(fp.getvalue()), item) @@ -279,6 +275,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. fp.seek(0) self.assertEqual(marshal.load(fp), item) @@ -314,6 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(fp.getvalue(), expected) def test_header_export_all(self): @@ -345,6 +343,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual( output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n" ) @@ -429,6 +428,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): @@ -536,6 +536,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) @@ -550,6 +551,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) self.assertEqual(exported, item) @@ -575,6 +577,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual( exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] @@ -586,6 +589,20 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def test_two_dict_items(self): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph\xa3", age="22") + i2 = TestItem( + name="Maria", age=1j + ) # Invalid datetimes didn't consistently fail between Python versions + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(TypeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + self.assertEqual(exported, [dict(i1), dict(i3)]) + def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") i2 = self.item_class(name="Maria", age=i1) @@ -593,6 +610,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = { "name": "Jesus", @@ -607,6 +625,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} self.assertEqual(exported, [expected]) @@ -616,11 +635,30 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) self.assertEqual(exported, [item]) +class JsonItemExporterToBytesTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + kwargs["encoding"] = "latin" + return JsonItemExporter(self.output, **kwargs) + + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph", age="22") + i2 = TestItem(name="\u263a", age="11") + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) + self.assertEqual(exported, [dict(i1), dict(i3)]) + + class JsonItemExporterDataclassTest(JsonItemExporterTest): item_class = TestDataClass custom_field_item_class = CustomFieldDataclass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b1059099a..6b82974fa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -36,6 +36,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FeedSlot, @@ -124,6 +125,9 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): @@ -236,8 +240,10 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - def test_parse_credentials(self): + def setUp(self): skip_if_no_boto() + + def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", "AWS_SECRET_ACCESS_KEY": "settings_secret", @@ -273,8 +279,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - skip_if_no_boto() - settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -286,30 +290,39 @@ class S3FeedStorageTest(unittest.TestCase): verifyObject(IFeedStorage, storage) file = mock.MagicMock() - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) + if IS_BOTO3_AVAILABLE: + storage.s3_client = mock.MagicMock() yield storage.store(file) - - stub.assert_no_pending_responses() self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), ) + else: + from botocore.stub import Stubber + + with Stubber(storage.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "Body": file, + "Bucket": bucket, + "Key": key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + mock.call.close(), + ], + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -336,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + def test_from_crawler_without_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -364,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + def test_from_crawler_with_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -391,9 +431,22 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + @defer.inlineCallbacks - def test_store_botocore_without_acl(self): - skip_if_no_boto() + def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -405,11 +458,18 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1]) + if IS_BOTO3_AVAILABLE: + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + else: + acl = storage.s3_client.put_object.call_args[1].get("ACL") + self.assertIsNone(acl) @defer.inlineCallbacks - def test_store_botocore_with_acl(self): - skip_if_no_boto() + def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -419,9 +479,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertEqual( - storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl" - ) + if IS_BOTO3_AVAILABLE: + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + else: + acl = storage.s3_client.put_object.call_args[1]["ACL"] + self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): with LogCapture() as log: @@ -726,10 +788,9 @@ class FeedExportTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[feed_options["format"]] = Path(file_path).read_bytes() + content[feed_options["format"]] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -889,15 +950,10 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_stats_multiple_file(self): settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", "FEEDS": { printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, - "s3://bucket/key/foo.csv": { - "format": "csv", - }, "stdout:": { "format": "xml", }, @@ -909,18 +965,12 @@ class FeedExportTest(FeedExportTestBase): self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) - self.assertIn( - "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats() - ) self.assertIn( "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1 - ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 ) @@ -946,9 +996,10 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": { self._random_temp_filename(): {"format": fmt}, }, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) - self.assertEqual(b"", data[fmt]) + self.assertEqual(None, data[fmt]) @defer.inlineCallbacks def test_start_finish_exporting_items(self): @@ -1064,8 +1115,7 @@ class FeedExportTest(FeedExportTestBase): with LogCapture() as log: yield self.exported_no_data(settings) - print(log) - self.assertEqual(str(log).count("Storage.store is called"), 3) + self.assertEqual(str(log).count("Storage.store is called"), 0) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -1271,6 +1321,17 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) + @defer.inlineCallbacks + def test_export_tuple(self): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux"}, + ] + + settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} + rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] + yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + @defer.inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields @@ -1732,10 +1793,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[str(file_path)] = Path(file_path).read_bytes() + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -2236,6 +2296,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue for file in sorted(dir_name.iterdir()): content[feed["format"]].append(file.read_bytes()) finally: @@ -2419,10 +2482,11 @@ class BatchDeliveriesTest(FeedExportTestBase): / self._file_mark: {"format": fmt}, }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(b"", data[fmt][0]) + self.assertEqual(0, len(data[fmt])) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -2536,9 +2600,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) - @pytest.mark.skipif( - sys.platform == "win32", reason="Odd behaviour on file creation/output" - ) @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2560,7 +2621,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len([_ for _ in data["json"] if _])) + self.assertEqual(len(items), len(data["json"])) @defer.inlineCallbacks def test_stats_batch_file_success(self): @@ -2587,7 +2648,6 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() - bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2647,7 +2707,7 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(TestSpider, settings) yield crawler.crawl() - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() @@ -2752,6 +2812,31 @@ class FeedExportInitTest(unittest.TestCase): with self.assertRaises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_absolute_pathlib_as_uri(self): + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + Path(tmp.name).resolve(): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + + def test_relative_pathlib_as_uri(self): + settings = { + "FEEDS": { + Path("./items.json"): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): def __init__(self, uri): @@ -2836,6 +2921,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None + def setUp(self): + skip_if_no_boto() + def test_init(self): settings_dict = { "FEED_URI": "file:///tmp/foobar", @@ -2901,8 +2989,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, } with pytest.warns( ScrapyDeprecationWarning, @@ -2923,8 +3011,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_from_crawler(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, } with pytest.warns( ScrapyDeprecationWarning, @@ -2990,10 +3078,7 @@ class URIParamsTest: spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with pytest.warns( - ScrapyDeprecationWarning, match="Modifying the params dictionary in-place" - ): - feed_exporter.open_spider(spider) + feed_exporter.open_spider(spider) self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") diff --git a/tests/test_mail.py b/tests/test_mail.py index bc7298e9d..504c78486 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import unittest from email.charset import Charset from io import BytesIO diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c80666586..bf96f17b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import dataclasses import os import random +import sys import time from datetime import datetime from io import BytesIO @@ -11,6 +12,7 @@ from unittest import mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -32,6 +34,7 @@ from scrapy.utils.test import ( get_gcs_content_and_delete, skip_if_no_boto, ) +from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func @@ -225,12 +228,16 @@ class FilesPipelineTestCase(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: + def setUp(self): + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + def test_item_fields_default(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings({"FILES_STORE": "s3://example/files/"}) - ) + pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] @@ -245,7 +252,7 @@ class FilesPipelineTestCaseFieldsMixin: pipeline = FilesPipeline.from_settings( Settings( { - "FILES_STORE": "s3://example/files/", + "FILES_STORE": self.tempdir, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", } @@ -461,8 +468,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.files_result_field, "this") - self.assertEqual(pipeline.files_urls_field, "that") + self.assertEqual( + pipeline.files_result_field, + UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, @@ -636,34 +648,35 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - uri = os.environ.get("FTP_TEST_FILE_URI") - if not uri: - raise unittest.SkipTest("No FTP URI available for testing") data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} path = "full/filename" - store = FTPFilesStore(uri) - empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) - yield store.persist_file(path, buf, info=None, meta=meta, headers=None) - stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") - path = f"{store.basedir}/{path}" - content = get_ftp_content_and_delete( - path, - store.host, - store.port, - store.username, - store.password, - store.USE_ACTIVE_MODE, - ) - self.assertEqual(data.decode(), content) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + empty_dict = yield store.stat_file(path, info=None) + self.assertEqual(empty_dict, {}) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + stat = yield store.stat_file(path, info=None) + self.assertIn("last_modified", stat) + self.assertIn("checksum", stat) + self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + path = f"{store.basedir}/{path}" + content = get_ftp_content_and_delete( + path, + store.host, + store.port, + store.username, + store.password, + store.USE_ACTIVE_MODE, + ) + self.assertEqual(data, content) class ItemWithFiles(Item): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6fd8e6308..8924875d1 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -598,8 +598,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.images_result_field, "something_else") - self.assertEqual(pipeline.images_urls_field, "something") + self.assertEqual( + pipeline.images_result_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.images_urls_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of ImagePipeline, diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index c05f4da91..dc0a82086 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -21,8 +21,6 @@ class MitmProxy: auth_pass = "scrapy" def start(self): - from scrapy.utils.test import get_testenv - script = """ import sys from mitmproxy.tools.main import mitmdump @@ -46,7 +44,6 @@ sys.exit(mitmdump()) "--ssl-insecure", ], stdout=PIPE, - env=get_testenv(), ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 8665a9205..7312eb036 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,10 +1,6 @@ -import sys import unittest -import warnings -from contextlib import suppress from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -162,30 +158,6 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): - def _assert_serializes_ok(self, request, spider=None): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - with suppress(KeyError): - del sys.modules[ - "scrapy.utils.reqser" - ] # delete module to reset the deprecation warning - - from scrapy.utils.reqser import request_from_dict as _from_dict - from scrapy.utils.reqser import request_to_dict as _to_dict - - request_copy = _from_dict(_to_dict(request, spider), spider) - self._assert_same_request(request, request_copy) - - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead", - str(caught[0].message), - ) - - class TestSpiderMixin: def __mixin_callback(self, response): pass diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 8d87a322a..d7a923085 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,3 @@ -# coding=utf-8 from twisted.trial import unittest diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4a577cd8c..e7799737f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +import pytest + from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -199,6 +201,21 @@ class BaseSettingsTest(unittest.TestCase): settings.update({"key_lowprio": 3}, priority=20) self.assertEqual(settings["key_lowprio"], 1) + @pytest.mark.xfail( + raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" + ) + def test_update_kwargs(self): + settings = BaseSettings({"key": 0}) + settings.update(key=1) # pylint: disable=unexpected-keyword-arg + + @pytest.mark.xfail( + raises=AttributeError, + reason="BaseSettings.update doesn't support iterable input", + ) + def test_update_iterable(self): + settings = BaseSettings({"key": 0}) + settings.update([("key", 1)]) + def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') @@ -217,6 +234,10 @@ class BaseSettingsTest(unittest.TestCase): self.assertIn("key_highprio", settings) del settings["key_highprio"] self.assertNotIn("key_highprio", settings) + with self.assertRaises(KeyError): + settings.delete("notkey") + with self.assertRaises(KeyError): + del settings["notkey"] def test_get(self): test_configuration = { @@ -451,6 +472,31 @@ class SettingsTest(unittest.TestCase): self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) + def test_pop_item_with_default_value(self): + settings = Settings() + + with self.assertRaises(KeyError): + settings.pop("DUMMY_CONFIG") + + dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") + self.assertEqual(dummy_config_value, "dummy_value") + + def test_pop_item_with_immutable_settings(self): + settings = Settings( + {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} + ) + + self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + + settings.freeze() + + with self.assertRaises(TypeError) as error: + settings.pop("OTHER_DUMMY_CONFIG") + + self.assertEqual( + str(error.exception), "Trying to modify an immutable Settings object" + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f645f4cce..f97125b76 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ import shutil -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from twisted.trial import unittest @@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase): Path(jobdir).mkdir() try: spider = Spider(name="default") - dt = datetime.now() + dt = datetime.now(tz=timezone.utc) ss = SpiderState(jobdir) ss.spider_opened(spider) diff --git a/tests/test_stats.py b/tests/test_stats.py index 7a8adf638..3d4c7e88e 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase): @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector(self, mock_datetime): fixed_datetime = datetime(2019, 12, 1, 11, 38) - mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime) + mock_datetime.now = mock.Mock(return_value=fixed_datetime) self.crawler.stats = StatsCollector(self.crawler) ext = CoreStats.from_crawler(self.crawler) ext.spider_opened(self.spider) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 01d0ee043..65e352053 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -4,6 +4,7 @@ from unittest import TestCase from pytest import mark +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -29,6 +30,8 @@ class AsyncioTest(TestCase): assert original_reactor == reactor + @mark.only_asyncio() + @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop() is asyncio.get_running_loop() + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 78ed9a7c9..dc3f01d57 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,6 +1,8 @@ import unittest import warnings +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -21,44 +23,45 @@ class BuildComponentListTest(unittest.TestCase): def test_backward_compatible_build_dict(self): base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} custom = {"two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(base, custom, convert=lambda x: x), + ["one", "four", "five", "three"], + ) def test_return_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, convert=lambda x: x), custom + ) def test_map_dict(self): custom = {"one": 1, "two": 2, "three": 3} - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list({}, custom, convert=lambda x: x.upper()), + ["ONE", "TWO", "THREE"], + ) def test_map_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] + ) def test_duplicate_components_in_dict(self): duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - self.assertRaises( - ValueError, - build_component_list, - {}, - duplicate_dict, - convert=lambda x: x.lower(), - ) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ["a", "b", "a"] with self.assertRaises(ValueError) as cm: - build_component_list(None, duplicate_list, convert=lambda x: x) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list(None, duplicate_list, convert=lambda x: x) self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): @@ -76,9 +79,8 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - self.assertRaises( - ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower() - ) + with self.assertRaises(ValueError): + build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): # work well with None and numeric values @@ -92,15 +94,9 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) # raise exception for invalid values d = {"one": "5"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "1.0"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": [1, 2, 3]} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": {"a": "a", "b": 2}} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "lorem ipsum"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b6a84ee91..9e5f88f48 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,9 +1,13 @@ import copy import unittest +import warnings from collections.abc import Mapping, MutableMapping +from typing import Iterator +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.datatypes import ( + CaseInsensitiveDict, CaselessDict, LocalCache, LocalWeakReferencedCache, @@ -14,16 +18,16 @@ from scrapy.utils.python import garbage_collect __doctests__ = ["scrapy.utils.datatypes"] -class CaselessDictTest(unittest.TestCase): +class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {"red": 1, "black": 3} - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -42,7 +46,7 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) @@ -67,12 +71,12 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMutableMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d["red"], 1) self.assertEqual(d["black"], 3) def test_caseless(self): - d = CaselessDict() + d = self.dict_class() d["key_Lower"] = 1 self.assertEqual(d["KEy_loWer"], 1) self.assertEqual(d.get("KEy_loWer"), 1) @@ -82,7 +86,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key_Lower"), 3) def test_delete(self): - d = CaselessDict({"key_lower": 1}) + d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] self.assertRaises(KeyError, d.__getitem__, "key_LOWER") self.assertRaises(KeyError, d.__getitem__, "key_lower") @@ -107,15 +111,15 @@ class CaselessDictTest(unittest.TestCase): def test_fromkeys(self): keys = ("a", "b") - d = CaselessDict.fromkeys(keys) + d = self.dict_class.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) - d = CaselessDict.fromkeys(keys, 1) + d = self.dict_class.fromkeys(keys, 1) self.assertEqual(d["A"], 1) self.assertEqual(d["B"], 1) - instance = CaselessDict() + instance = self.dict_class() d = instance.fromkeys(keys) self.assertEqual(d["A"], None) self.assertEqual(d["B"], None) @@ -125,31 +129,35 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d["B"], 1) def test_contains(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 - assert "a" in d + assert "A" in d def test_pop(self): - d = CaselessDict() + d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) self.assertRaises(KeyError, d.pop, "A") def test_normkey(self): - class MyDict(CaselessDict): - def normkey(self, key): + class MyDict(self.dict_class): + def _normkey(self, key): return key.title() + normkey = _normkey # deprecated CaselessDict class + d = MyDict() d["key-one"] = 2 self.assertEqual(list(d.keys()), ["Key-One"]) def test_normvalue(self): - class MyDict(CaselessDict): - def normvalue(self, value): + class MyDict(self.dict_class): + def _normvalue(self, value): if value is not None: return value + 1 + normvalue = _normvalue # deprecated CaselessDict class + d = MyDict({"key": 1}) self.assertEqual(d["key"], 2) self.assertEqual(d.get("key"), 2) @@ -174,11 +182,51 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get("key"), 2) def test_copy(self): - h1 = CaselessDict({"header1": "value"}) + h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) + assert isinstance(h2, self.dict_class) self.assertEqual(h1, h2) self.assertEqual(h1.get("header1"), h2.get("header1")) - assert isinstance(h2, CaselessDict) + self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + h3 = h1.copy() + assert isinstance(h3, self.dict_class) + self.assertEqual(h1, h3) + self.assertEqual(h1.get("header1"), h3.get("header1")) + self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + + +class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaseInsensitiveDict + + def test_repr(self): + d1 = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d1), "") + d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + self.assertEqual( + repr(d2), "" + ) + + def test_iter(self): + d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + iterkeys = iter(d) + self.assertIsInstance(iterkeys, Iterator) + self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + + +class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaselessDict + + def test_deprecation_message(self): + with warnings.catch_warnings(record=True) as caught: + self.dict_class({"foo": "bar"}) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + str(caught[0].message), + ) class SequenceExcludeTest(unittest.TestCase): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 2d9210410..eedb6f6af 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -296,3 +296,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath("scrapy.unmatched.Path") self.assertEqual(output, "scrapy.unmatched.Path") self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring) diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 6b2a458bc..7b7a25db8 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase): def test_gunzip_no_gzip_file_raises(self): self.assertRaises( - IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() + OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() ) def test_gunzip_truncated_short(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index faf7d2709..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) + self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") diff --git a/tox.ini b/tox.ini index af8f1f57a..3ed8b6f63 100644 --- a/tox.ini +++ b/tox.ini @@ -16,10 +16,6 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' - # Extras - botocore>=1.4.87 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -37,15 +33,18 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.2.0 + mypy==1.4.1 + typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==9.4.0.19 - types-Pygments==2.14.0.7 - types-pyOpenSSL==23.1.0.1 - types-setuptools==67.6.0.7 + types-Pillow==10.0.0.2 + types-Pygments==2.15.0.2 + types-pyOpenSSL==23.2.0.2 + types-setuptools==68.0.0.3 + # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 + w3lib >= 2.1.2 commands = - mypy --show-error-codes {posargs: scrapy tests} + mypy {posargs: scrapy tests} [testenv:pre-commit] basepython = python3 @@ -58,7 +57,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.2 + pylint==2.17.5 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -73,7 +72,7 @@ commands = [pinned] deps = - cryptography==3.4.6 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -85,16 +84,11 @@ deps = Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.3.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -103,7 +97,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -126,14 +120,27 @@ commands = {[pinned]commands} basepython = python3 deps = {[testenv]deps} - boto + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + boto3==1.20.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = {[pinned]commands} [testenv:asyncio] commands = @@ -193,3 +200,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}