diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3c1c8f891..956c512cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.1 commit = True tag = True tag_name = {new_version} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..28771216c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: Run test suite +on: [push, pull_request] + +jobs: + test-windows: + name: "Windows Tests" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest] + python-version: [3.7, 3.8] + env: [TOXENV: py] + include: + - os: windows-latest + python-version: 3.6 + env: + TOXENV: windows-pinned + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Run test suite + env: ${{ matrix.env }} + run: | + pip install -U tox twine wheel codecov + tox diff --git a/.travis.yml b/.travis.yml index b883c5b78..880c8772b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,10 +23,12 @@ matrix: - env: TOXENV=asyncio-pinned python: 3.6.1 - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 + dist: bionic - env: TOXENV=py python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + dist: bionic - env: TOXENV=py python: 3.7 diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c03e258c7..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - TOXENV: py -pool: - vmImage: 'windows-latest' -strategy: - matrix: - Python36: - python.version: '3.6' - TOXENV: windows-pinned - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' -- script: | - pip install -U tox twine wheel codecov - tox - displayName: 'Run test suite' diff --git a/docs/conf.py b/docs/conf.py index 27d2b5dff..543507a46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3bfd3bc3b..73d7ede42 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -69,10 +69,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -265,7 +264,6 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com diff --git a/docs/news.rst b/docs/news.rst index 850b323ef..0391506c4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,333 @@ Release notes ============= +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import path of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-11) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + .. _release-2.3.0: Scrapy 2.3.0 (2020-08-04) @@ -4008,9 +4335,9 @@ First release of Scrapy. .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index c04044e8f..82c5f271f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -1,3 +1,5 @@ +.. _using-asyncio: + ======= asyncio ======= diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..6801adc9c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -207,6 +207,11 @@ CookiesMiddleware a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` @@ -684,11 +689,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ef50c9f5c..0a0a1765a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -123,7 +123,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': return f'$ {str(value)}' - return super(Product, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) .. _topics-exporters-reference: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 14096ada4..519f18b63 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -257,6 +257,12 @@ settings: * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` +.. note:: + + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + requests) are still processed. + .. setting:: CLOSESPIDER_TIMEOUT CLOSESPIDER_TIMEOUT @@ -279,8 +285,6 @@ Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. -Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1744cfd74..843ed25f9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ + * Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -319,6 +319,8 @@ For instance:: }, } +.. _feed-options: + The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition: @@ -329,6 +331,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. @@ -337,6 +341,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. + .. versionadded:: 2.4.0 + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). @@ -355,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + .. versionadded:: 2.4.0 + - ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. @@ -517,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT FEED_EXPORT_BATCH_ITEM_COUNT ------------------------------ +---------------------------- + +.. versionadded:: 2.3.0 Default: ``0`` @@ -586,11 +596,15 @@ The function signature should be as follows: If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` is always ``1``. + .. versionadded:: 2.3.0 + - ``batch_time``: UTC date and time, in ISO format with ``:`` replaced with ``-``. See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``time``: ``batch_time``, with microseconds set to ``0``. :type params: dict diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 06809c24b..156897274 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -56,6 +56,8 @@ this: error will be logged and the file won't be present in the ``files`` field. +.. _images-pipeline: + Using the Images Pipeline ========================= @@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to -JPEG/RGB format, so you need to install this library in order to use it. -`Python Imaging Library`_ (PIL) should also work in most cases, but it is known -to cause troubles in some setups, so we recommend to use `Pillow`_ instead of -PIL. +The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow -.. _Python Imaging Library: http://www.pythonware.com/products/pil/ .. _topics-media-pipeline-enabling: @@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses the passive connection mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +.. _media-pipelines-s3: + Amazon S3 storage ----------------- .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 -bucket. Scrapy will automatically upload the files to the bucket. +If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will +automatically upload the files to the bucket. For example, this is a valid :setting:`IMAGES_STORE` value:: @@ -187,8 +188,9 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like -self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: +You can also use other S3-like storages. Storages like self-hosted `Minio`_ or +`s3.scality`_. All you need to do is set endpoint option in you Scrapy +settings:: AWS_ENDPOINT_URL = 'http://minio.example.com:9000' @@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +.. _botocore: https://github.com/boto/botocore +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _s3.scality: https://s3.scality.com/ -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _media-pipeline-gcs: @@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 30b1945d0..98906992d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -61,6 +61,12 @@ Request objects :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type headers: dict :param cookies: the request cookies. These can be sent in two forms. @@ -102,6 +108,12 @@ Request objects ) For more info see :ref:`cookies-mw`. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). @@ -681,9 +693,19 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + :type protocol: :class:`str` + + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: VERSION + The ``protocol`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -768,6 +790,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -783,6 +807,17 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: VERSION + + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 218b23c87..b948dbfde 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section. Import paths and classes ======================== -.. versionadded:: VERSION +.. versionadded:: 2.4.0 When a setting references a callable object to be imported by Scrapy, such as a class or a function, there are two different ways you can specify that object: @@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP Default: ``None`` -Import path of a given asyncio event loop class. +Import path of a given ``asyncio`` event loop class. -If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the -asyncio event loop to be used with it. Set the setting to the import path of the +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the desired asyncio event loop class. If the setting is set to ``None`` the default asyncio event loop will be used. If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` -function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop -class to be used. +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. +.. caution:: Please be aware that, when using a non-default event loop + (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with + :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call + :func:`asyncio.set_event_loop`, which will set the specified event loop + as the current loop for the current OS thread. + .. setting:: BOT_NAME BOT_NAME @@ -352,6 +358,11 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. +.. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + .. setting:: DEPTH_LIMIT DEPTH_LIMIT diff --git a/pytest.ini b/pytest.ini index ca8191f42..d4deeb57c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +xfail_strict = true usefixtures = chdir python_files=test_*.py __init__.py python_classes= @@ -35,9 +36,5 @@ flake8-ignore = scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/utils/http.py F403 - scrapy/utils/markup.py F403 - scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - diff --git a/scrapy/VERSION b/scrapy/VERSION index 276cbf9e2..005119baa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.3.0 +2.4.1 diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 72248bded..5f44daa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -98,7 +98,7 @@ class Command(ScrapyCommand): print(f"Created spider {name!r} using template {template_name!r} ", end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n {spiders_module.__name__}.{module}") + print(f"in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): template_file = join(self.templates_dir, f'{template}.tmpl') diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index b7a803152..c9154bec1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -411,6 +411,11 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError, IndexError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -419,6 +424,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..9524cce2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a18c26b17..9ce823dbc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,14 +1,10 @@ import os import json import logging -import warnings from os.path import join, exists -from queuelib import PriorityQueue - from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir -from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) @@ -56,14 +52,6 @@ class Scheduler: dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - if pqclass is PriorityQueue: - warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" - " is no longer supported because of API changes; " - "please use 'scrapy.pqueues.ScrapyPriorityQueue'", - ScrapyDeprecationWarning) - from scrapy.pqueues import ScrapyPriorityQueue - pqclass = ScrapyPriorityQueue - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('SCHEDULER_DEBUG') diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 87f8152a4..d95ed3d38 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -97,35 +97,14 @@ class CookiesMiddleware: def _get_request_cookies(self, jar, request): """ - Extract cookies from a Request. Values from the `Request.cookies` attribute - take precedence over values from the `Cookie` request header. + Extract cookies from the Request.cookies attribute """ - def get_cookies_from_header(jar, request): - cookie_header = request.headers.get("Cookie") - if not cookie_header: - return [] - cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) - cookie_list_unicode = [] - for cookie_bytes in cookie_gen_bytes: - try: - cookie_unicode = cookie_bytes.decode("utf8") - except UnicodeDecodeError: - logger.warning("Non UTF-8 encoded cookie found in request %s: %s", - request, cookie_bytes) - cookie_unicode = cookie_bytes.decode("latin1", errors="replace") - cookie_list_unicode.append(cookie_unicode) - response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) - return jar.make_cookies(response, request) - - def get_cookies_from_attribute(jar, request): - if not request.cookies: - return [] - elif isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies - formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) - response = Response(request.url, headers={"Set-Cookie": formatted}) - return jar.make_cookies(response, request) - - return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..f504302e2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,3 +1,4 @@ +import io import zlib from scrapy.utils.gz import gunzip @@ -14,6 +15,12 @@ try: except ImportError: pass +try: + import zstandard + ACCEPTED_ENCODINGS.append(b'zstd') +except ImportError: + pass + class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be @@ -67,4 +74,9 @@ class HttpCompressionMiddleware: body = zlib.decompress(body, -15) if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) + if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + # Using its streaming API since its simple API could handle only cases + # where there is content size data embedded in the frame + reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) + body = reader.read() return body diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7dcb2f52e..bec114707 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -319,18 +319,26 @@ class FeedExporter: # Use `largs=log_args` to copy log_args into function's scope # instead of using `log_args` from the outer scope d.addCallback( - lambda _, largs=log_args: logger.info( - logfmt % "Stored", largs, extra={'spider': spider} - ) + self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ ) d.addErrback( - lambda f, largs=log_args: logger.error( - logfmt % "Error storing", largs, - exc_info=failure_to_exc_info(f), extra={'spider': spider} - ) + self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ ) return d + def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") + + def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. @@ -452,7 +460,7 @@ class FeedExporter: crawler = getattr(self, 'crawler', None) def build_instance(builder, *preargs): - return build_storage(builder, uri, preargs=preargs) + return build_storage(builder, uri, feed_options=feed_options, preargs=preargs) if crawler and hasattr(feedcls, 'from_crawler'): instance = build_instance(feedcls.from_crawler, crawler) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ef58deacc..498f1b052 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -65,7 +65,11 @@ class Request(object_ref): s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ('://' not in self._url) and (not self._url.startswith('data:')): + if ( + '://' not in self._url + and not self._url.startswith('about:') + and not self._url.startswith('data:') + ): raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2815303a2..7f267c800 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -160,7 +160,7 @@ def _select_value(ele, n, v): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected - # And for select tags wihout options + # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) elif v is not None and multiple: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..185a9bb67 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +37,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -89,8 +100,9 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + for x in [ + "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", + ]: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0a12f3e2c..0c2ee6856 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -86,7 +86,7 @@ class MediaPipeline: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=1) + dfd = DeferredList(dlist, consumeErrors=True) return dfd.addCallback(self.item_completed, item, info) def _process_request(self, request, info, item): diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f191deac6..0bef555a6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from twisted.internet import defer from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple +from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -50,6 +50,36 @@ class CachingThreadedResolver(ThreadedResolver): return result +@implementer(IHostResolution) +class HostResolution: + def __init__(self, name): + self.name = name + + def cancel(self): + raise NotImplementedError() + + +@provider(IResolutionReceiver) +class _CachingResolutionReceiver: + def __init__(self, resolutionReceiver, hostName): + self.resolutionReceiver = resolutionReceiver + self.hostName = hostName + self.addresses = [] + + def resolutionBegan(self, resolution): + self.resolutionReceiver.resolutionBegan(resolution) + self.resolution = resolution + + def addressResolved(self, address): + self.resolutionReceiver.addressResolved(address) + self.addresses.append(address) + + def resolutionComplete(self): + self.resolutionReceiver.resolutionComplete() + if self.addresses: + dnscache[self.hostName] = self.addresses + + @implementer(IHostnameResolver) class CachingHostnameResolver: """ @@ -73,33 +103,22 @@ class CachingHostnameResolver: def install_on_reactor(self): self.reactor.installNameResolver(self) - def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, - addressTypes=None, transportSemantics='TCP'): - - @provider(IResolutionReceiver) - class CachingResolutionReceiver(resolutionReceiver): - - def resolutionBegan(self, resolution): - super().resolutionBegan(resolution) - self.resolution = resolution - self.resolved = False - - def addressResolved(self, address): - super().addressResolved(address) - self.resolved = True - - def resolutionComplete(self): - super().resolutionComplete() - if self.resolved: - dnscache[hostName] = self.resolution - + def resolveHostName( + self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP" + ): try: - return dnscache[hostName] + addresses = dnscache[hostName] except KeyError: return self.original_resolver.resolveHostName( - CachingResolutionReceiver(), + _CachingResolutionReceiver(resolutionReceiver, hostName), hostName, portNumber, addressTypes, - transportSemantics + transportSemantics, ) + else: + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py new file mode 100644 index 000000000..7f697af5f --- /dev/null +++ b/scrapy/utils/asyncgen.py @@ -0,0 +1,5 @@ +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21ba02a0b..6db9cc117 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -105,7 +105,7 @@ def process_parallel(callbacks, input, *a, **kw): callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 11d433cf5..76156a4b8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ +import struct from gzip import GzipFile from io import BytesIO -import re -import struct from scrapy.utils.decorators import deprecated @@ -42,17 +41,5 @@ def gunzip(data): return b''.join(output_list) -_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search -_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search - - -@deprecated -def is_gzipped(response): - """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - cenc = response.headers.get('Content-Encoding', b'').lower() - return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') - - def gzip_magic_number(response): return response.body[:3] == b'\x1f\x8b\x08' diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py deleted file mode 100644 index ceb3f0509..000000000 --- a/scrapy/utils/http.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.http instead of this module -""" - -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.decorators import deprecated -from w3lib.http import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.http` is deprecated, " - "Please import from `w3lib.http` instead.", - ScrapyDeprecationWarning, stacklevel=2) - - -@deprecated -def decode_chunked_transfer(chunked_body): - """Parsed body received with chunked transfer encoding, and return the - decoded body. - - For more info see: - https://en.wikipedia.org/wiki/Chunked_transfer_encoding - - """ - body, h, t = '', '', chunked_body - while t: - h, t = t.split('\r\n', 1) - if h == '0': - break - size = int(h, 16) - body += t[:size] - t = t[size + 2:] - return body diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py deleted file mode 100644 index 9728c542a..000000000 --- a/scrapy/utils/markup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.html instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.markup` is deprecated. " - "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py deleted file mode 100644 index 5dcf791b8..000000000 --- a/scrapy/utils/multipart.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.form instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.multipart` is deprecated. " - "If you're using `encode_multipart` function, please use " - "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fd13d85e3..c66af497e 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,4 @@ import os -import pickle import warnings from importlib import import_module @@ -68,18 +67,10 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - if pickled_settings: - warnings.warn("Use of environment variable " - "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' " - "is deprecated.", ScrapyDeprecationWarning) - settings.setdict(pickle.loads(pickled_settings), priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', - 'PICKLED_SETTINGS_TO_OVERRIDE', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index c8c24076e..653e2bbbb 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,10 +1,11 @@ -""" -Helpers using Python 3.6+ syntax (ignore SyntaxError on import). -""" +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401 -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results +warnings.warn( + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, +) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 831d29462..6723d9b37 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None): if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.new_event_loop() + event_loop = asyncio.get_event_loop() asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f3a9a67a3..59fc9202f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,17 +4,14 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.py36 import collect_asyncgen -except SyntaxError: - collect_asyncgen = None +from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/setup.cfg b/setup.cfg index 8101443e3..0c9f6b963 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,9 +55,6 @@ ignore_errors = True [mypy-scrapy.utils.response] ignore_errors = True -[mypy-scrapy.utils.spider] -ignore_errors = True - [mypy-scrapy.utils.trackref] ignore_errors = True diff --git a/setup.py b/setup.py index e9dd30162..a95014c98 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', @@ -35,11 +34,12 @@ install_requires = [ 'h2>=3.2.0', ] extras_require = {} - +cpython_dependencies = [ + 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by # https://foss.heptapod.net/pypy/pypy/-/issues/2498, @@ -50,14 +50,14 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py deleted file mode 100644 index 2c466da04..000000000 --- a/tests/CrawlerProcess/alternative_name_resolver.py +++ /dev/null @@ -1,15 +0,0 @@ -import scrapy -from scrapy.crawler import CrawlerProcess - - -class IPv6Spider(scrapy.Spider): - name = "ipv6_spider" - start_urls = ["http://[::1]"] - - -process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", -}) -process.crawl(IPv6Spider) -process.start() diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..46c2a12a4 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,44 @@ +import asyncio +import sys + +from scrapy import Spider +from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import deferred_from_coro +from twisted.internet.defer import Deferred + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + return deferred_from_coro(self._open_spider(spider)) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + }) + process.crawl(UrlSpider) + process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..f9eab3543 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,30 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + name = "caching_hostname_resolver_spider" + + def start_requests(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..3340d2f84 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index 60d91b68b..05a98fbec 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ name = "ipv6_spider" start_urls = ["http://[::1]"] -process = CrawlerProcess(settings={"RETRY_ENABLED": False}) -process.crawl(IPv6Spider) -process.start() +if __name__ == "__main__": + process = CrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..68b856a88 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,12 +1,9 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 -pytest-azurepipelines pytest-cov pytest-twisted >= 1.11 pytest-xdist @@ -16,6 +13,7 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython -brotlipy +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin new file mode 100644 index 000000000..b5c2038e8 Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-content-size.bin differ diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin new file mode 100644 index 000000000..3d494192e Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin differ diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin new file mode 100644 index 000000000..97bdbcae0 Binary files /dev/null and b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin differ diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e54a0948..d3ac05eac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' + spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) @@ -680,9 +681,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ @@ -813,9 +819,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 85035a220..ab113710d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from tests.mockserver import MockServer + class BaseCrawlerTest(unittest.TestCase): @@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name): + def run_script(self, script_name, *script_args): script_path = os.path.join(self.script_dir, script_name) - args = (sys.executable, script_path) + args = [sys.executable, script_path] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -321,11 +323,20 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) - def test_ipv6_alternative_name_resolver(self): - log = self.run_script('alternative_name_resolver.py') - self.assertIn('Spider closed (finished)', log) + def test_caching_hostname_resolver_ipv6(self): + log = self.run_script("caching_hostname_resolver_ipv6.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_caching_hostname_resolver_finite_execution(self): + with MockServer() as mock_server: + http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") + log = self.run_script("caching_hostname_resolver.py", http_address) + self.assertIn("Spider closed (finished)", log) + self.assertNotIn("ERROR: Error downloading", log) + self.assertNotIn("TimeoutError", log) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) @@ -353,6 +364,25 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertIn("Using asyncio event loop: uvloop.Loop", log) + @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") + @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..93e7311d2 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('_SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index abbb262f9..86d72772c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -114,6 +114,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -368,6 +369,13 @@ class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls: Type = HTTP10DownloadHandler + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + class Https10TestCase(Http10TestCase): scheme = 'https' @@ -497,6 +505,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.1") + return d + class Https11TestCase(Http11TestCase): scheme = 'https' @@ -971,7 +986,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) - + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -1133,3 +1148,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..b538a0ed3 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,11 +1,13 @@ import asyncio -from unittest import mock +from unittest import mock, SkipTest from pytest import mark +from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -211,10 +213,21 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using AsyncIO and Twisted versions lower than ' + '18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: @@ -235,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using Twisted versions lower than 18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index a3de307ee..aff8542e9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,6 +2,8 @@ import logging from testfixtures import LogCapture from unittest import TestCase +import pytest + from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.exceptions import NotConfigured @@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase): self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) @@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) @@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..4c5bfc577 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -20,6 +20,12 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin + 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin + 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin + 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), } @@ -80,6 +86,27 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b"') + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) - def test_is_x_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/x-gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_not_quite(self): - hdrs = Headers({"Content-Type": "application/gzippppp"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_case_insensitive(self): - hdrs = Headers({"Content-Type": "Application/X-Gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_wrong(self): - hdrs = Headers({"Content-Type": "application/javascript"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_with_charset(self): - hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py deleted file mode 100644 index 363b015a8..000000000 --- a/tests/test_utils_http.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from scrapy.utils.http import decode_chunked_transfer - - -class ChunkedTest(unittest.TestCase): - - def test_decode_chunked_transfer(self): - """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" - chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" - chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" - chunked_body += "3\r\n" + "con\r\n" - chunked_body += "8\r\n" + "sequence\r\n" - chunked_body += "0\r\n\r\n" - body = decode_chunked_transfer(chunked_body) - self.assertEqual( - body, - "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" - ) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..ad7394232 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,14 @@ import asyncio +from unittest import SkipTest +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure +from twisted import version as twisted_version from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.python.versions import Version +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using AsyncIO and Twisted ' + 'versions lower than 18.4.0' + ) + + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using Twisted versions lower ' + 'than 18.4.0' + ) + + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a60181a3a..f935a8689 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,7 +4,10 @@ Tests borrowed from the twisted.web.client tests. """ import os import shutil +import sys +from pkg_resources import parse_version +import cryptography import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -414,6 +417,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): + if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): + self.skipTest("This does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) diff --git a/tox.ini b/tox.ini index b165abd1b..a1451ae66 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ minversion = 1.7.0 deps = -ctests/constraints.txt -rtests/requirements-py3.txt + # mitmproxy does not support PyPy + # mitmproxy does not support Windows when running Python < 3.7 + mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 Pillow>=4.0.0 @@ -21,6 +25,8 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} @@ -68,7 +74,6 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 @@ -76,16 +81,31 @@ deps = w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 +install_command = + # --use-feature=2020-resolver is required, otherwise the latest verion of + # Twisted gets installed. + pip install --use-feature=2020-resolver {opts} {packages} +setenv = + _SCRAPY_PINNED=true [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 - + PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} +setenv = + {[pinned]setenv} + [testenv:windows-pinned] basepython = python3 deps = @@ -93,20 +113,33 @@ deps = # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 + PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = {[testenv]deps} reppy robotexclusionrulesparser +install_command = + # Test --use-feature=2020-resolver for the latest version of all + # dependencies. + pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] -commands = {[testenv:asyncio]commands} deps = {[testenv:pinned]deps} +install_command = + {[pinned]install_command} +commands = {[testenv:asyncio]commands} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 @@ -115,11 +148,15 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 +install_command = + {[pinned]install_command} +commands = {[testenv:pypy3]commands} +setenv = + {[pinned]setenv} [docs] changedir = docs