diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 956c512cb..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.1 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d781ae81..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,5 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 543507a46..406c4d94a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -183,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ diff --git a/docs/intro/install.rst b/docs/intro/install.rst index bf919ce25..8581dde0b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -12,6 +12,7 @@ Supported Python versions Scrapy requires Python 3.6+, either the CPython implementation (default) or the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +.. _intro-install-scrapy: Installing Scrapy ================= @@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -268,4 +290,6 @@ For details, see `Issue #2473 `_. .. _zsh: https://www.zsh.org/ .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/news.rst b/docs/news.rst index d9fe897ad..0ea412e75 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,190 @@ Release notes ============= +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + .. _release-2.4.1: Scrapy 2.4.1 (2020-11-17) @@ -97,6 +281,8 @@ Backward-incompatible changes (:issue:`4717`, :issue:`4823`) +.. _2.4-deprecation-removals: + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -1433,6 +1619,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -4184,7 +4374,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -4331,6 +4521,7 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ diff --git a/docs/pip.txt b/docs/pip.txt deleted file mode 100644 index 095e53a0d..000000000 --- a/docs/pip.txt +++ /dev/null @@ -1,3 +0,0 @@ -# In pip 20.3-21.0, the default dependency resolver causes the build in -# ReadTheDocs to fail due to memory exhaustion or timeout. -pip<20.3 diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d34b47da..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ Sphinx>=3.0 sphinx-hoverxref>=0.2b1 sphinx-notfound-page>=0.4 -sphinx_rtd_theme>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6801adc9c..b539c23df 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -892,6 +892,11 @@ settings (see the settings documentation for more info): If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 2f1517906..9150ca7d9 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -64,10 +64,10 @@ NotConfigured This exception can be raised by some components to indicate that they will remain disabled. Those components include: - * Extensions - * Item pipelines - * Downloader middlewares - * Spider middlewares +- Extensions +- Item pipelines +- Downloader middlewares +- Spider middlewares The exception must be raised in the component's ``__init__`` method. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 519f18b63..9e86fd0fe 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -7,8 +7,7 @@ Extensions The extensions framework provides a mechanism for inserting your own custom functionality into Scrapy. -Extensions are just regular classes that are instantiated at Scrapy startup, -when extensions are initialized. +Extensions are just regular classes. Extension settings ================== @@ -27,8 +26,8 @@ Loading & activating extensions =============================== Extensions are loaded and activated at startup by instantiating a single -instance of the extension class. Therefore, all the extension initialization -code must be performed in the class ``__init__`` method. +instance of the extension class per spider being run. All the extension +initialization code must be performed in the class ``__init__`` method. To make an extension available, add it to the :setting:`EXTENSIONS` setting in your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 843ed25f9..e772a461c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -21,10 +21,10 @@ Serialization formats For serializing the scraped data, the feed exports use the :ref:`Item exporters `. These formats are supported out of the box: - * :ref:`topics-feed-format-json` - * :ref:`topics-feed-format-jsonlines` - * :ref:`topics-feed-format-csv` - * :ref:`topics-feed-format-xml` +- :ref:`topics-feed-format-json` +- :ref:`topics-feed-format-jsonlines` +- :ref:`topics-feed-format-csv` +- :ref:`topics-feed-format-xml` But you can also extend the supported format through the :setting:`FEED_EXPORTERS` setting. @@ -34,54 +34,58 @@ But you can also extend the supported format through the JSON ---- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` - * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` - * See :ref:`this warning ` if you're using JSON with - large feeds. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` + +- Exporter used: :class:`~scrapy.exporters.JsonItemExporter` + +- See :ref:`this warning ` if you're using JSON with + large feeds. .. _topics-feed-format-jsonlines: JSON lines ---------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` - * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` +- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: CSV --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` - * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use - :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this - option, but it is important for CSV because unlike many other export - formats CSV uses a fixed header. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` + +- Exporter used: :class:`~scrapy.exporters.CsvItemExporter` + +- To specify columns to export and their order use + :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this + option, but it is important for CSV because unlike many other export + formats CSV uses a fixed header. .. _topics-feed-format-xml: XML --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` - * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` +- Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: Pickle ------ - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` - * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` +- Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: Marshal ------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` - * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` +- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` .. _topics-feed-storage: @@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: - * :ref:`topics-feed-storage-fs` - * :ref:`topics-feed-storage-ftp` - * :ref:`topics-feed-storage-s3` (requires botocore_) - * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - * :ref:`topics-feed-storage-stdout` +- :ref:`topics-feed-storage-fs` +- :ref:`topics-feed-storage-ftp` +- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) +- :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are not available. For example, the S3 backend is only available if the botocore_ @@ -114,8 +118,8 @@ Storage URI parameters The storage URI can also contain parameters that get replaced when the feed is being created. These parameters are: - * ``%(time)s`` - gets replaced by a timestamp when the feed is being created - * ``%(name)s`` - gets replaced by the spider name +- ``%(time)s`` - gets replaced by a timestamp when the feed is being created +- ``%(name)s`` - gets replaced by the spider name Any other named parameter gets replaced by the spider attribute of the same name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id`` @@ -123,13 +127,13 @@ attribute the moment the feed is being created. Here are some examples to illustrate: - * Store in FTP using one directory per spider: +- Store in FTP using one directory per spider: - * ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` + - ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` - * Store in S3 using one directory per spider: +- Store in S3 using one directory per spider: - * ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` + - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` .. _topics-feed-storage-backends: @@ -144,9 +148,9 @@ Local filesystem The feeds are stored in the local filesystem. - * URI scheme: ``file`` - * Example URI: ``file:///tmp/export.csv`` - * Required external libraries: none +- URI scheme: ``file`` +- Example URI: ``file:///tmp/export.csv`` +- Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if you specify an absolute path like ``/tmp/export.csv``. This only works on Unix @@ -159,9 +163,9 @@ FTP The feeds are stored in a FTP server. - * URI scheme: ``ftp`` - * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` - * Required external libraries: none +- URI scheme: ``ftp`` +- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` +- Required external libraries: none FTP supports two different connection modes: `active or passive `_. Scrapy uses the passive connection @@ -178,23 +182,25 @@ S3 The feeds are stored on `Amazon S3`_. - * URI scheme: ``s3`` - * Example URIs: +- URI scheme: ``s3`` - * ``s3://mybucket/path/to/export.csv`` - * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `botocore`_ >= 1.4.87 + - ``s3://mybucket/path/to/export.csv`` + + - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` + +- Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: - * :setting:`AWS_ACCESS_KEY_ID` - * :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_ACCESS_KEY_ID` +- :setting:`AWS_SECRET_ACCESS_KEY` You can also define a custom ACL for exported feeds using this setting: - * :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_STORAGE_S3_ACL` This storage backend uses :ref:`delayed file delivery `. @@ -208,19 +214,20 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gs`` - * Example URIs: +- URI scheme: ``gs`` - * ``gs://mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `google-cloud-storage`_. + - ``gs://mybucket/path/to/export.csv`` + +- Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: - * :setting:`FEED_STORAGE_GCS_ACL` - * :setting:`GCS_PROJECT_ID` +- :setting:`FEED_STORAGE_GCS_ACL` +- :setting:`GCS_PROJECT_ID` This storage backend uses :ref:`delayed file delivery `. @@ -234,9 +241,9 @@ Standard output The feeds are written to the standard output of the Scrapy process. - * URI scheme: ``stdout`` - * Example URI: ``stdout:`` - * Required external libraries: none +- URI scheme: ``stdout`` +- Example URI: ``stdout:`` +- Required external libraries: none .. _delayed-file-delivery: @@ -264,16 +271,16 @@ Settings These are the settings used for configuring the feed exports: - * :setting:`FEEDS` (mandatory) - * :setting:`FEED_EXPORT_ENCODING` - * :setting:`FEED_STORE_EMPTY` - * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT` - * :setting:`FEED_STORAGES` - * :setting:`FEED_STORAGE_FTP_ACTIVE` - * :setting:`FEED_STORAGE_S3_ACL` - * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +- :setting:`FEEDS` (mandatory) +- :setting:`FEED_EXPORT_ENCODING` +- :setting:`FEED_STORE_EMPTY` +- :setting:`FEED_EXPORT_FIELDS` +- :setting:`FEED_EXPORT_INDENT` +- :setting:`FEED_STORAGES` +- :setting:`FEED_STORAGE_FTP_ACTIVE` +- :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index c3445d40e..00806392a 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method:: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. +You can also filter log records by :class:`~logging.LogRecord` data. For +example, you can filter log records by message content using a substring or +a regular expression. Create a :class:`logging.Filter` subclass +and equip it with a regular expression pattern to +filter out unwanted messages:: + + import logging + import re + + class ContentFilter(logging.Filter): + def filter(self, record): + match = re.search(r'\d{3} [Ee]rror, retrying', record.message) + if match: + return False + +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to +filter all loggers in different parts of the project +(middlewares, spider, etc.):: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + for handler in logging.root.handlers: + handler.addFilter(ContentFilter()) + +Alternatively, you may choose a specific logger +and hide it without affecting other loggers:: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('my_logger') + logger.addFilter(ContentFilter()) + scrapy.utils.log module ======================= diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c0283df01..500781c05 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -703,7 +703,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The ``protocol`` parameter. .. attribute:: Response.url @@ -809,7 +809,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The protocol that was used to download the response. For instance: "HTTP/1.0", "HTTP/1.1" diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c7ec2e0cc..9caba5ee5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to achieve this, such as: - * `BeautifulSoup`_ is a very popular web scraping library among Python - programmers which constructs a Python object based on the structure of the - HTML code and also deals with bad markup reasonably well, but it has one - drawback: it's slow. +- `BeautifulSoup`_ is a very popular web scraping library among Python + programmers which constructs a Python object based on the structure of the + HTML code and also deals with bad markup reasonably well, but it has one + drawback: it's slow. - * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic - API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard - library.) +- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic + API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python + standard library.) Scrapy comes with its own mechanism for extracting data. They're called selectors because they "select" certain parts of the HTML document specified diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0a4684a91..1d5babcec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE Default:: { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', @@ -677,6 +678,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +.. _http2: + The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update :setting:`DOWNLOAD_HANDLERS` as follows:: @@ -703,7 +706,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. - - No support for the :signal:`bytes_received` signal. + - No support for the :signal:`bytes_received` and + :signal:`headers_received` signals. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption @@ -1229,20 +1233,6 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. -.. setting:: RETRY_PRIORITY_ADJUST - -RETRY_PRIORITY_ADJUST ---------------------- - -Default: ``-1`` - -Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` - -Adjust retry request priority relative to original request: - -- a positive priority adjust means higher priority. -- **a negative priority adjust (default) means lower priority.** - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 0f46f1c87..b910fc453 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -95,20 +95,21 @@ convenience. Available Shortcuts ------------------- - * ``shelp()`` - print a help with the list of available objects and shortcuts +- ``shelp()`` - print a help with the list of available objects and + shortcuts - * ``fetch(url[, redirect=True])`` - fetch a new response from the given - URL and update all related objects accordingly. You can optionaly ask for - HTTP 3xx redirections to not be followed by passing ``redirect=False`` +- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL + and update all related objects accordingly. You can optionaly ask for HTTP + 3xx redirections to not be followed by passing ``redirect=False`` - * ``fetch(request)`` - fetch a new response from the given request and - update all related objects accordingly. +- ``fetch(request)`` - fetch a new response from the given request and update + all related objects accordingly. - * ``view(response)`` - open the given response in your local web browser, for - inspection. This will add a `\ tag`_ to the response body in order - for external links (such as images and style sheets) to display properly. - Note, however, that this will create a temporary file in your computer, - which won't be removed automatically. +- ``view(response)`` - open the given response in your local web browser, for + inspection. This will add a `\ tag`_ to the response body in order + for external links (such as images and style sheets) to display properly. + Note, however, that this will create a temporary file in your computer, + which won't be removed automatically. .. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base @@ -122,21 +123,21 @@ content). Those objects are: - * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. +- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for - the current URL +- ``spider`` - the Spider which is known to handle the URL, or a + :class:`~scrapy.spiders.Spider` object if there is no spider found for the + current URL - * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched - page. You can modify this request using :meth:`~scrapy.http.Request.replace` - or fetch a new request (without leaving the shell) using the ``fetch`` - shortcut. +- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched + page. You can modify this request using + :meth:`~scrapy.http.Request.replace` or fetch a new request (without + leaving the shell) using the ``fetch`` shortcut. - * ``response`` - a :class:`~scrapy.http.Response` object containing the last - fetched page +- ``response`` - a :class:`~scrapy.http.Response` object containing the last + fetched page - * ``settings`` - the current :ref:`Scrapy settings ` +- ``settings`` - the current :ref:`Scrapy settings ` Example of shell session ======================== diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 98cfa606c..3d838fb63 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -403,7 +403,7 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.5 .. signal:: headers_received .. function:: headers_received(headers, request, spider) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index d87b89292..7c0d4a635 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -260,7 +260,8 @@ this:: The ``handle_httpstatus_list`` key of :attr:`Request.meta ` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` -to ``True`` if you want to allow any response code for a request. +to ``True`` if you want to allow any response code for a request, and ``False`` to +disable the effects of the ``handle_httpstatus_all`` key. Keep in mind, however, that it's usually a bad idea to handle non-200 responses, unless you really know what you're doing. diff --git a/scrapy/VERSION b/scrapy/VERSION index 005119baa..437459cd9 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.1 +2.5.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4326ca4aa..8a8065bf2 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -22,7 +22,7 @@ __all__ = [ # Scrapy and Twisted versions -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) twisted_version = (_txv.major, _txv.minor, _txv.micro) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 23ccffcd9..6e77551c6 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,8 @@ Base class for Scrapy commands """ import os from optparse import OptionGroup +from typing import Any, Dict + from twisted.python import failure from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli @@ -15,7 +17,7 @@ class ScrapyCommand: crawler_process = None # default settings to be used for this command instead of global defaults - default_settings = {} + default_settings: Dict[str, Any] = {} exitcode = 0 diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 999c987ea..6bdf9eae0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' + url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f205c40b0..0f2a21b85 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is not supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 83ee074da..52118db1b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,6 @@ import json import logging +from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url @@ -10,6 +11,7 @@ from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError + logger = logging.getLogger(__name__) @@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items = {} - requests = {} + items: Dict[int, list] = {} + requests: Dict[int, list] = {} first_response = None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index db0a56e56..b47e55092 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,16 +1,77 @@ -import sys import re +import sys from functools import wraps from inspect import getmembers +from typing import Dict from unittest import TestCase from scrapy.http import Request -from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec +from scrapy.utils.spider import iterate_spider_output + + +class Contract: + """ Abstract class for contracts """ + request_cls = None + + def __init__(self, method, *args): + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.args = args + + def add_pre_hook(self, request, results): + if hasattr(self, 'pre_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + try: + results.startTest(self.testcase_pre) + self.pre_process(response) + results.stopTest(self.testcase_pre) + except AssertionError: + results.addFailure(self.testcase_pre, sys.exc_info()) + except Exception: + results.addError(self.testcase_pre, sys.exc_info()) + else: + results.addSuccess(self.testcase_pre) + finally: + return list(iterate_spider_output(cb(response, **cb_kwargs))) + + request.callback = wrapper + + return request + + def add_post_hook(self, request, results): + if hasattr(self, 'post_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) + try: + results.startTest(self.testcase_post) + self.post_process(output) + results.stopTest(self.testcase_post) + except AssertionError: + results.addFailure(self.testcase_post, sys.exc_info()) + except Exception: + results.addError(self.testcase_post, sys.exc_info()) + else: + results.addSuccess(self.testcase_post) + finally: + return output + + request.callback = wrapper + + return request + + def adjust_request_args(self, args): + return args class ContractsManager: - contracts = {} + contracts: Dict[str, Contract] = {} def __init__(self, contracts): for contract in contracts: @@ -107,66 +168,6 @@ class ContractsManager: request.errback = eb_wrapper -class Contract: - """ Abstract class for contracts """ - request_cls = None - - def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') - self.args = args - - def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - try: - results.startTest(self.testcase_pre) - self.pre_process(response) - results.stopTest(self.testcase_pre) - except AssertionError: - results.addFailure(self.testcase_pre, sys.exc_info()) - except Exception: - results.addError(self.testcase_pre, sys.exc_info()) - else: - results.addSuccess(self.testcase_pre) - finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) - - request.callback = wrapper - - return request - - def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) - try: - results.startTest(self.testcase_post) - self.post_process(output) - results.stopTest(self.testcase_post) - except AssertionError: - results.addFailure(self.testcase_post, sys.exc_info()) - except Exception: - results.addError(self.testcase_post, sys.exc_info()) - else: - results.addSuccess(self.testcase_post) - finally: - return output - - request.callback = wrapper - - return request - - def adjust_request_args(self, args): - return args - - def _create_testcase(method, desc): spider = method.__self__.name diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..a5619d8a4 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,12 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from twisted.internet import defer +from typing import Callable, Union +from twisted.internet import defer +from twisted.python.failure import Failure + +from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager @@ -29,15 +33,14 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_exception'): self.methods['process_exception'].appendleft(mw.process_exception) - def download(self, download_func, request, spider): + def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks - def process_request(request): + def process_request(request: Request): for method in self.methods['process_request']: response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_request must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" ) if response: @@ -45,7 +48,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return (yield download_func(request=request, spider=spider)) @defer.inlineCallbacks - def process_response(response): + def process_response(response: Union[Response, Request]): if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -55,8 +58,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_response must return Response or Request, " + f"Middleware {method.__qualname__} must return Response or Request, " f"got {type(response)}" ) if isinstance(response, Request): @@ -64,14 +66,13 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response @defer.inlineCallbacks - def process_exception(failure): + def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_exception must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" ) if response: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7eedbf33e..8ce57af2f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,12 +3,14 @@ extracts information from them""" import collections import logging from collections import deque +from collections.abc import Iterable +from typing import Union from itemadapter import is_item from twisted.internet import defer from twisted.python.failure import Failure -from scrapy import signals +from scrapy import signals, Spider from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -90,28 +92,26 @@ class Scraper: def close_spider(self, spider): """Close a spider being scraped and release its resources""" - slot = self.slot - slot.closing = defer.Deferred() - slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider, slot) - return slot.closing + self.slot.closing = defer.Deferred() + self.slot.closing.addCallback(self.itemproc.close_spider) + self._check_if_closing(spider) + return self.slot.closing def is_idle(self): """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider, slot): - if slot.closing and slot.is_idle(): - slot.closing.callback(spider) + def _check_if_closing(self, spider): + if self.slot.closing and self.slot.is_idle(): + self.slot.closing.callback(spider) def enqueue_scrape(self, response, request, spider): - slot = self.slot - dfd = slot.add_response_request(response, request) + dfd = self.slot.add_response_request(response, request) def finish_scraping(_): - slot.finish_response(response, request) - self._check_if_closing(spider, slot) - self._scrape_next(spider, slot) + self.slot.finish_response(response, request) + self._check_if_closing(spider) + self._scrape_next(spider) return _ dfd.addBoth(finish_scraping) @@ -120,15 +120,15 @@ class Scraper: {'request': request}, exc_info=failure_to_exc_info(f), extra={'spider': spider})) - self._scrape_next(spider, slot) + self._scrape_next(spider) return dfd - def _scrape_next(self, spider, slot): - while slot.queue: - response, request, deferred = slot.next_response_request_deferred() + def _scrape_next(self, spider): + while self.slot.queue: + response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result, request, spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the downloaded response or failure through the spider callback/errback """ @@ -139,7 +139,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result, request, spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the different cases of request's result been a Response or a Failure """ @@ -149,7 +149,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result, request, spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -164,7 +164,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, response, spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -185,7 +185,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result, request, response, spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) if isinstance(result, collections.abc.AsyncIterable): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d0d292007..6fcbc7492 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,10 +5,14 @@ See documentation in docs/topics/spider-middleware.rst """ import collections.abc from itertools import islice +from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable +from twisted.internet.defer import Deferred from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput +from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.conf import build_component_list @@ -16,8 +20,11 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableAsyncChain, MutableChain -def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') +ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] + + +def _isiterable(o) -> bool: + return isinstance(o, (Iterable, AsyncIterable)) class SpiderMiddlewareManager(MiddlewareManager): @@ -39,7 +46,8 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func, response, request, spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Any: for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -53,7 +61,8 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, + exception_processor_index: int, recover_to: MutableChain) -> Generator: @_process_iterable_universal async def _evaluate_async_iterable(iterable): try: @@ -67,7 +76,8 @@ class SpiderMiddlewareManager(MiddlewareManager): recover_to.extend(exception_result) return _evaluate_async_iterable(iterable) - def _process_spider_exception(self, response, spider, _failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, + start_index: int = 0) -> Union[Failure, MutableChain]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -89,7 +99,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure - def _process_spider_output(self, response, spider, result, start_index=0): + def _process_spider_output(self, response: Response, spider: Spider, + result: Iterable, start_index: int = 0) -> MutableChain: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method last_result_async = isinstance(result, collections.abc.AsyncIterator) @@ -125,7 +136,7 @@ class SpiderMiddlewareManager(MiddlewareManager): else: return MutableChain(result, recovered) - def _process_callback_output(self, response, spider, result): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: if isinstance(result, collections.abc.AsyncIterator): recovered = MutableAsyncChain() else: @@ -137,16 +148,17 @@ class SpiderMiddlewareManager(MiddlewareManager): else: return MutableChain(result, recovered) - def scrape_response(self, scrape_func, response, request, spider): - def process_callback_output(result): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Deferred: + def process_callback_output(result: Iterable) -> MutableChain: return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure): + def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider): + def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f504302e2..4e7feeeaf 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,10 +1,12 @@ import io +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -25,11 +27,25 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats=None): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + try: + return cls(stats=crawler.stats) + except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -44,6 +60,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 51fe59254..5965a1c6c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,7 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -import logging +from logging import getLogger, Logger +from typing import Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -23,12 +24,104 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.exceptions import NotConfigured -from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.exceptions import NotConfigured +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.python import global_object_name +from scrapy.utils.response import response_status_message -logger = logging.getLogger(__name__) + +retry_logger = getLogger(__name__) + + +def get_retry_request( + request: Request, + *, + spider: Spider, + reason: Union[str, Exception] = 'unspecified', + max_retry_times: Optional[int] = None, + priority_adjust: Optional[int] = None, + logger: Logger = retry_logger, + stats_base_key: str = 'retry', +): + """ + Returns a new :class:`~scrapy.Request` object to retry the specified + request, or ``None`` if retries of the specified request have been + exhausted. + + For example, in a :class:`~scrapy.Spider` callback, you could use it as + follows:: + + def parse(self, response): + if not response.text: + new_request_or_none = get_retry_request( + response.request, + spider=self, + reason='empty', + ) + return new_request_or_none + + *spider* is the :class:`~scrapy.Spider` instance which is asking for the + retry request. It is used to access the :ref:`settings ` + and :ref:`stats `, and to provide extra logging context (see + :func:`logging.debug`). + + *reason* is a string or an :class:`Exception` object that indicates the + reason why the request needs to be retried. It is used to name retry stats. + + *max_retry_times* is a number that determines the maximum number of times + that *request* can be retried. If not specified or ``None``, the number is + read from the :reqmeta:`max_retry_times` meta key of the request. If the + :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number + is read from the :setting:`RETRY_TIMES` setting. + + *priority_adjust* is a number that determines how the priority of the new + request changes in relation to *request*. If not specified, the number is + read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + + *logger* is the logging.Logger object to be used when logging messages + + *stats_base_key* is a string to be used as the base key for the + retry-related job stats + """ + settings = spider.crawler.settings + stats = spider.crawler.stats + retry_times = request.meta.get('retry_times', 0) + 1 + if max_retry_times is None: + max_retry_times = request.meta.get('max_retry_times') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= max_retry_times: + logger.debug( + "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider} + ) + new_request = request.copy() + new_request.meta['retry_times'] = retry_times + new_request.dont_filter = True + if priority_adjust is None: + priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + new_request.priority = request.priority + priority_adjust + + if callable(reason): + reason = reason() + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value(f'{stats_base_key}/count') + stats.inc_value(f'{stats_base_key}/reason_count/{reason}') + return new_request + else: + stats.inc_value(f'{stats_base_key}/max_reached') + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) + return None class RetryMiddleware: @@ -67,31 +160,12 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - retries = request.meta.get('retry_times', 0) + 1 - - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] - - stats = spider.crawler.stats - if retries <= retry_times: - logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) - retryreq = request.copy() - retryreq.meta['retry_times'] = retries - retryreq.dont_filter = True - retryreq.priority = request.priority + self.priority_adjust - - if isinstance(reason, Exception): - reason = global_object_name(reason.__class__) - - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') - return retryreq - else: - stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) + max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) + priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) + return get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0c97e6999..bf4ae7b45 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,10 +1,16 @@ +import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +# Defined in the http.cookiejar module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 +IPV4_RE = re.compile(r"\.\d+$", re.ASCII) + + class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7f267c800..ef2eb3ba6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -71,7 +71,7 @@ def _urlencode(seq, enc): values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] - return urlencode(values, doseq=1) + return urlencode(values, doseq=True) def _get_form(response, formname, formid, formnumber, formxpath): diff --git a/scrapy/item.py b/scrapy/item.py index af3849302..2ccd7ad18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -8,6 +8,7 @@ from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat +from typing import Dict from warnings import warn from scrapy.utils.deprecate import ScrapyDeprecationWarning @@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta): class DictItem(MutableMapping, BaseItem): - fields = {} + fields: Dict[str, Field] = {} def __new__(cls, *args, **kwargs): if issubclass(cls, DictItem) and not issubclass(cls, Item): diff --git a/scrapy/mail.py b/scrapy/mail.py index 7d7a2c435..2a25ccd44 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,7 +9,7 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText -from email.utils import COMMASPACE, formatdate +from email.utils import formatdate from io import BytesIO from twisted.internet import defer, ssl @@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +# Defined in the email.utils module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42 +COMMASPACE = ", " + + def _to_bytes_or_none(text): if text is None: return None diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 5040378ea..bbec38086 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,8 +1,13 @@ -from collections import defaultdict, deque import logging import pprint +from collections import defaultdict, deque +from typing import Callable, Deque, Dict +from twisted.internet.defer import Deferred + +from scrapy import Spider from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object from scrapy.utils.defer import process_parallel, process_chain, process_chain_both @@ -16,16 +21,16 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(deque) + self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> list: raise NotImplementedError @classmethod - def from_settings(cls, settings, crawler=None): + def from_settings(cls, settings: Settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -52,24 +57,24 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw): + def _add_middleware(self, mw) -> None: if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname, obj, *args): + def _process_parallel(self, methodname: str, obj, *args) -> Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname, obj, *args): + def _process_chain(self, methodname: str, obj, *args) -> Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: return self._process_parallel('close_spider', spider) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index ae5c258df..9861456de 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -32,7 +32,7 @@ class HttpErrorMiddleware: if 200 <= response.status < 300: # common case return meta = response.meta - if 'handle_httpstatus_all' in meta: + if meta.get('handle_httpstatus_all', False): return if 'handle_httpstatus_list' in meta: allowed_statuses = meta['handle_httpstatus_list'] diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8d862d1d0..e0a22592f 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,16 +3,17 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ import warnings +from typing import Tuple from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy.http import Request, Response -from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.exceptions import NotConfigured +from scrapy.http import Request, Response from scrapy.utils.asyncgen import _process_iterable_universal -from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -31,7 +32,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES = LOCAL_SCHEMES + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + name: str def referrer(self, response_url, request_url): raise NotImplementedError() @@ -89,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = POLICY_NO_REFERRER + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): return None @@ -109,7 +111,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): if not self.tls_protected(response_url) or self.tls_protected(request_url): @@ -126,7 +128,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_SAME_ORIGIN + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): if self.origin(response_url) == self.origin(request_url): @@ -142,7 +144,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): return self.origin_referrer(response_url) @@ -161,7 +163,7 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( @@ -182,7 +184,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -209,7 +211,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -235,7 +237,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = POLICY_UNSAFE_URL + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): return self.stripped_referrer(response_url) @@ -247,8 +249,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') - name = POLICY_SCRAPY_DEFAULT + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + name: str = POLICY_SCRAPY_DEFAULT _policy_classes = {p.name: p for p in ( diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 6c0bb1d10..92118ddb9 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,9 +1,10 @@ import collections import functools import inspect +from collections.abc import AsyncIterable -async def collect_asyncgen(result): +async def collect_asyncgen(result: AsyncIterable): results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ca3d79fa6..f81faf6dd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,16 +3,21 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect +from collections.abc import Coroutine from functools import wraps +from typing import Any, Callable, Generator, Iterable -from twisted.internet import defer, task +from twisted.internet import defer +from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.task import Cooperator from twisted.python import failure +from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure): +def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -20,12 +25,12 @@ def defer_fail(_failure): before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result): +def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -33,13 +38,13 @@ def defer_succeed(result): before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result): - if isinstance(result, defer.Deferred): +def defer_result(result) -> Deferred: + if isinstance(result, Deferred): return result elif isinstance(result, failure.Failure): return defer_fail(result) @@ -47,7 +52,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -64,15 +69,15 @@ def mustbe_deferred(f, *args, **kw): return defer_result(result) -def parallel(iterable, count, callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ - coop = task.Cooperator() + coop = Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return DeferredList([coop.coiterate(work) for _ in range(count)]) class _AsyncCooperatorAdapter: @@ -172,24 +177,24 @@ class _AsyncCooperatorAdapter: def parallel_async(async_iterable, count, callable, *args, **named): """ Like parallel but for async iterables """ - coop = task.Cooperator() + coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)]) return dl -def process_chain(callbacks, input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = defer.Deferred() + d = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both(callbacks, errbacks, input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = defer.Deferred() + d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, errback=eb, @@ -203,17 +208,17 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): return d -def process_parallel(callbacks, input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d -def iter_errback(iterable, errback, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -241,22 +246,22 @@ async def aiter_errback(aiterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o): +def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - if isinstance(o, defer.Deferred): + if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return defer.ensureDeferred(o) + return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return defer.Deferred.fromFuture(asyncio.ensure_future(o)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o -def deferred_f_from_coro_f(coro_f): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -268,14 +273,14 @@ def deferred_f_from_coro_f(coro_f): return f -def maybeDeferred_coro(f, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + return defer.fail(failure.Failure(captureVars=Deferred.debug)) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): return result elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index c8d4391b1..a90f1d278 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,13 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -import weakref -from urllib.parse import urlparse +from typing import Union +from urllib.parse import urlparse, ParseResult +from weakref import WeakKeyDictionary + +from scrapy.http import Request, Response -_urlparse_cache = weakref.WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() -def urlparse_cached(request_or_response): +def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0bf9bff70..8b823d174 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,6 +8,7 @@ import re import sys import warnings import weakref +from collections.abc import Iterable from functools import partial, wraps from itertools import chain @@ -336,15 +337,15 @@ else: gc.collect() -class MutableChain: +class MutableChain(Iterable): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args): + def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables): + def extend(self, *iterables: Iterable): self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 12c03d78e..66736b42f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,20 +4,27 @@ scrapy.http.Request objects """ import hashlib -import weakref +from typing import Dict, Iterable, Optional, Tuple, Union from urllib.parse import urlunparse +from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url +from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache = weakref.WeakKeyDictionary() +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_fingerprint_cache = WeakKeyDictionary() -def request_fingerprint(request, include_headers=None, keep_fragments=False): +def request_fingerprint( + request: Request, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +): """ Return the request fingerprint. @@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): (for instance when handling requests with a headless browser). """ + headers: Optional[Tuple[bytes, ...]] = None if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) + cache_key = (headers, keep_fragments) if cache_key not in cache: fp = hashlib.sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) fp.update(request.body or b'') - if include_headers: - for hdr in include_headers: + if headers: + for hdr in headers: if hdr in request.headers: fp.update(hdr) for v in request.headers.getlist(hdr): @@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): return cache[cache_key] -def request_authenticate(request, username, password): +def request_authenticate(request: Request, username: str, password: str) -> None: """Autenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) -def request_httprepr(request): +def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled @@ -92,7 +100,7 @@ def request_httprepr(request): return s -def referer_str(request): +def referer_str(request: Request) -> Optional[str]: """ Return Referer HTTP header suitable for logging. """ referrer = request.headers.get('Referer') if referrer is None: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 99b089b6f..b3ef7b463 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,19 +3,23 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import weakref import webbrowser import tempfile +from typing import Any, Callable, Iterable, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import scrapy +from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode from w3lib import html -_baseurl_cache = weakref.WeakKeyDictionary() +_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() -def get_base_url(response): +def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -23,10 +27,13 @@ def get_base_url(response): return _baseurl_cache[response] -_metaref_cache = weakref.WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() -def get_meta_refresh(response, ignore_tags=('script', 'noscript')): +def get_meta_refresh( + response: "scrapy.http.response.text.TextResponse", + ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), +) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): return _metaref_cache[response] -def response_status_message(status): +def response_status_message(status: Union[bytes, float, int, str]) -> str: """Return status code plus status text descriptive message """ - message = http.RESPONSES.get(int(status), "Unknown Status") - return f'{status} {to_unicode(message)}' + status_int = int(status) + message = http.RESPONSES.get(status_int, "Unknown Status") + return f'{status_int} {to_unicode(message)}' -def response_httprepr(response): +def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). @@ -60,7 +68,10 @@ def response_httprepr(response): return b"".join(values) -def open_in_browser(response, _openfunc=webbrowser.open): +def open_in_browser( + response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + _openfunc: Callable[[str], Any] = webbrowser.open, +) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 3e40acd69..b0c6a2424 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -import weakref -from time import time -from operator import itemgetter from collections import defaultdict +from operator import itemgetter +from time import time +from typing import DefaultDict +from weakref import WeakKeyDictionary NoneType = type(None) -live_refs = defaultdict(weakref.WeakKeyDictionary) +live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: diff --git a/setup.cfg b/setup.cfg index 0c9f6b963..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,54 +10,18 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy] -ignore_errors = True - -[mypy-scrapy.commands] -ignore_errors = True - -[mypy-scrapy.commands.parse] -ignore_errors = True - [mypy-scrapy.downloadermiddlewares.httpproxy] ignore_errors = True -[mypy-scrapy.contracts] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True -[mypy-scrapy.item] -ignore_errors = True - -[mypy-scrapy.http.cookies] -ignore_errors = True - -[mypy-scrapy.mail] -ignore_errors = True - [mypy-scrapy.pipelines.images] ignore_errors = True [mypy-scrapy.settings.default_settings] ignore_errors = True -[mypy-scrapy.spidermiddlewares.referer] -ignore_errors = True - -[mypy-scrapy.utils.httpobj] -ignore_errors = True - -[mypy-scrapy.utils.request] -ignore_errors = True - -[mypy-scrapy.utils.response] -ignore_errors = True - -[mypy-scrapy.utils.trackref] -ignore_errors = True - [mypy-tests.mocks.dummydbm] ignore_errors = True @@ -79,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/setup.py b/setup.py index 767c6f6bf..2b60a10af 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.2.0', + 'h2>=3.0,<4.0', ] extras_require = {} cpython_dependencies = [ diff --git a/tests/spiders.py b/tests/spiders.py index 78228ab04..81627f427 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -46,7 +46,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] def parse(self, response): @@ -281,7 +281,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4c5bfc577..40e9f3a96 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,13 +1,16 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -32,8 +35,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -56,6 +61,34 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -72,6 +105,20 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b' meta(max_retry_times) - self.mw.max_retry_times = 5 meta_max_retry_times = 4 + middleware_max_retry_times = 5 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_dont_retry(self): + max_retry_times = 4 + spider, middleware = self.get_spider_and_middleware() + meta = { + 'max_retry_times': max_retry_times, + 'dont_retry': True, + } + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + 0, + spider=spider, + middleware=middleware, + ) - # SETTINGS: meta(max_retry_times) = 4 - meta_max_retry_times = 4 - - req = Request(self.invalid_url, meta={ - 'max_retry_times': meta_max_retry_times, 'dont_retry': True - }) - - self._test_retry(req, DNSLookupError('foo'), 0) - - def _test_retry(self, req, exception, max_retry_times): + def _test_retry( + self, + req, + exception, + max_retry_times, + spider=None, + middleware=None, + ): + spider = spider or self.spider + middleware = middleware or self.mw for i in range(0, max_retry_times): - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) self.assertEqual(req, None) +class GetRetryRequestTest(unittest.TestCase): + + def get_spider(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider('foo') + + def test_basic_usage(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_max_retries_reached(self): + request = Request('https://example.com') + spider = self.get_spider() + max_retry_times = 0 + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_one_retry(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=1, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_two_retries(self): + spider = self.get_spider() + request = Request('https://example.com') + new_request = request + max_retry_times = 2 + for index in range(max_retry_times): + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = index + 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -expected_retry_times) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + value = spider.crawler.stats.get_value(stat) + self.assertEqual(value, expected_retry_times) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_no_spider(self): + request = Request('https://example.com') + with self.assertRaises(TypeError): + get_retry_request(request) # pylint: disable=missing-kwoa + + def test_max_retry_times_setting(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_meta(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_argument(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times + 1} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + + def test_priority_adjust_setting(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_priority_adjust_argument(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + priority_adjust=priority_adjust, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_log_extra_retry_success(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + get_retry_request( + request, + spider=spider, + ) + log.check_present(spider) + + def test_log_extra_retries_exceeded(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present(spider) + + def test_reason_string(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = 'because' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError() + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest() + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_custom_logger(self): + logger = logging.getLogger("custom-logger") + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + logger=logger, + ) + log.check_present( + ( + "custom-logger", + "DEBUG", + f"Retrying {request} (failed 1 times): {expected_reason}", + ) + ) + + def test_custom_stats_key(self): + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + stats_key = "custom_retry" + get_retry_request( + request, + spider=spider, + reason=expected_reason, + stats_base_key=stats_key, + ) + for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_engine.py b/tests/test_engine.py index ef1204f94..b2d1d83c7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -56,7 +56,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -66,15 +66,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -97,14 +97,10 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) class ItemZeroDivisionErrorSpider(TestSpider): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e449cd706..46f74ae52 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + def test_httperror_allow_all_false(self): + crawler = get_crawler(_HttpErrorSpider) + mw = HttpErrorMiddleware.from_crawler(crawler) + request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) + request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + res404 = self.res404.copy() + res404.request = request_httpstatus_false + res402 = self.res402.copy() + res402.request = request_httpstatus_true + + self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + self.assertIsNone(mw.process_spider_input(res402, self.spider)) + class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index f935a8689..6e4cb9b6e 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayloadDisabledCipher(self): if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This does work in PyPy with cryptography<=2.3.1") + self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) diff --git a/tox.ini b/tox.ini index 6907c8906..5b0606f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ basepython = python3 deps = mypy==0.780 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 @@ -71,7 +71,7 @@ commands = deps = cryptography==2.0 cssselect==0.9.1 - h2==3.2.0 + h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15