mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into typing-request-response-cls
This commit is contained in:
Andrey Rakhmatullin
commit
b6587575a1
36
.bandit.yml
36
.bandit.yml
|
|
@ -1,21 +1,19 @@
|
|||
skips:
|
||||
- B101
|
||||
- B113 # https://github.com/PyCQA/bandit/issues/1010
|
||||
- B105
|
||||
- B301
|
||||
- B303
|
||||
- B306
|
||||
- B307
|
||||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B324
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
- B406
|
||||
- B410
|
||||
- B503
|
||||
- B603
|
||||
- B605
|
||||
- B101 # assert_used
|
||||
- B105 # hardcoded_password_string
|
||||
- B301 # pickle
|
||||
- B307 # eval
|
||||
- B311 # random
|
||||
- B320 # xml_bad_etree
|
||||
- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
- B324 # hashlib "Use of weak SHA1 hash for security"
|
||||
- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403 # import_pickle
|
||||
- B404 # import_subprocess
|
||||
- B406 # import_xml_sax
|
||||
- B410 # import_lxml
|
||||
- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
|
||||
- B503 # ssl_with_bad_defaults
|
||||
- B603 # subprocess_without_shell_equals_true
|
||||
- B605 # start_process_with_a_shell
|
||||
exclude_dirs: ['tests']
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
[bumpversion]
|
||||
current_version = 2.11.0
|
||||
current_version = 2.11.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
||||
[bumpversion:file:scrapy/VERSION]
|
||||
|
||||
[bumpversion:file:SECURITY.md]
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.x
|
||||
serialize = {major}.{minor}.x
|
||||
|
|
|
|||
2
.flake8
2
.flake8
|
|
@ -1,7 +1,7 @@
|
|||
[flake8]
|
||||
|
||||
max-line-length = 119
|
||||
ignore = W503, E203
|
||||
ignore = E203, E501, E701, E704, W503
|
||||
|
||||
exclude =
|
||||
docs/conf.py
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
repos:
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.7.5
|
||||
rev: 1.7.7
|
||||
hooks:
|
||||
- id: bandit
|
||||
args: [-r, -c, .bandit.yml]
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 6.1.0
|
||||
rev: 7.0.0
|
||||
hooks:
|
||||
- id: flake8
|
||||
- repo: https://github.com/psf/black.git
|
||||
rev: 23.9.1
|
||||
rev: 24.2.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.12.0
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
- repo: https://github.com/adamchainz/blacken-docs
|
||||
|
|
@ -21,4 +21,4 @@ repos:
|
|||
hooks:
|
||||
- id: blacken-docs
|
||||
additional_dependencies:
|
||||
- black==23.9.1
|
||||
- black==24.2.0
|
||||
|
|
|
|||
11
README.rst
11
README.rst
|
|
@ -17,9 +17,10 @@ Scrapy
|
|||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
|
||||
:alt: Ubuntu
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
|
||||
:alt: macOS
|
||||
.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
|
||||
.. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
|
||||
.. :alt: macOS
|
||||
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
|
||||
|
|
@ -41,7 +42,7 @@ Scrapy
|
|||
Overview
|
||||
========
|
||||
|
||||
Scrapy is a fast high-level web crawling and web scraping framework, used to
|
||||
Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to
|
||||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
|
|
@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list.
|
|||
Commercial Support
|
||||
==================
|
||||
|
||||
See https://scrapy.org/support/ for details.
|
||||
See https://scrapy.org/support/ for details.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 2.11.x | :white_check_mark: |
|
||||
| < 2.11.x | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new.
|
||||
|
|
@ -273,7 +273,7 @@
|
|||
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||
</p>
|
||||
<p class="copyright">
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@
|
|||
Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
|
||||
</p>
|
||||
<p class="copyright">
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
|
||||
Made with <span class='sh-red'>❤</span> by <a href="https://www.zyte.com">Zyte</a>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
|
|
|
|||
|
|
@ -227,7 +227,7 @@ latex_documents = [
|
|||
# A list of regular expressions that match URIs that should not be checked when
|
||||
# doing a linkcheck build.
|
||||
linkcheck_ignore = [
|
||||
"http://localhost:\d+",
|
||||
r"http://localhost:\d+",
|
||||
"http://hg.scrapy.org",
|
||||
"http://directory.google.com/",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ Scrapy:
|
|||
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
|
||||
There is a hook in the pre-commit config
|
||||
that will automatically format your code before every commit. You can also
|
||||
run black manually with ``tox -e black``.
|
||||
run black manually with ``tox -e pre-commit``.
|
||||
|
||||
* Don't put your name in the code you contribute; git provides enough
|
||||
metadata to identify author of the code.
|
||||
|
|
|
|||
27
docs/faq.rst
27
docs/faq.rst
|
|
@ -297,9 +297,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and
|
|||
consume a lot of memory.
|
||||
|
||||
In order to avoid parsing all the entire feed at once in memory, you can use
|
||||
the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators``
|
||||
module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use
|
||||
under the cover.
|
||||
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
||||
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` uses.
|
||||
|
||||
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
||||
|
||||
.. autofunction:: scrapy.utils.iterators.csviter
|
||||
|
||||
Does Scrapy manage cookies automatically?
|
||||
-----------------------------------------
|
||||
|
|
@ -405,6 +409,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
|
|||
:ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
|
||||
|
||||
.. _faq-blank-request:
|
||||
|
||||
How can I make a blank request?
|
||||
-------------------------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
|
||||
blank_request = Request("data:,")
|
||||
|
||||
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
||||
in-line in web pages as if they were external resources. The "data:" scheme with an empty
|
||||
content (",") essentially creates a request to a data URL without any specific content.
|
||||
|
||||
|
||||
Running ``runspider`` I get ``error: No spider found in file: <filename>``
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
|
|
|
|||
143
docs/news.rst
143
docs/news.rst
|
|
@ -3,6 +3,105 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.11.1:
|
||||
|
||||
Scrapy 2.11.1 (2024-02-14)
|
||||
--------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
- Security bug fixes.
|
||||
|
||||
- Support for Twisted >= 23.8.0.
|
||||
|
||||
- Documentation improvements.
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Addressed `ReDoS vulnerabilities`_:
|
||||
|
||||
- ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of
|
||||
:func:`~scrapy.utils.iterators.xmliter_lxml`, which
|
||||
:class:`~scrapy.spiders.XMLFeedSpider` now uses.
|
||||
|
||||
To minimize the impact of this change on existing code,
|
||||
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
|
||||
the node namespace with a prefix in the node name, and big files with
|
||||
highly nested trees when using libxml2 2.7+.
|
||||
|
||||
- Fixed regular expressions in the implementation of the
|
||||
:func:`~scrapy.utils.response.open_in_browser` function.
|
||||
|
||||
Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information.
|
||||
|
||||
.. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS
|
||||
.. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9
|
||||
|
||||
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
|
||||
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
|
||||
advisory`_ for more information.
|
||||
|
||||
.. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7
|
||||
|
||||
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the
|
||||
deprecated ``scrapy.downloadermiddlewares.decompression`` module has been
|
||||
removed.
|
||||
|
||||
- The ``Authorization`` header is now dropped on redirects to a different
|
||||
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`,
|
||||
:issue:`6064`, :issue:`6142`)
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- The OS signal handling code was refactored to no longer use private Twisted
|
||||
functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`)
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization
|
||||
changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`)
|
||||
|
||||
- Extended documentation for :attr:`Request.meta <scrapy.http.Request.meta>`.
|
||||
(:issue:`5565`)
|
||||
|
||||
- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`,
|
||||
:issue:`6077`)
|
||||
|
||||
- Added a link to Zyte's export guides to the :ref:`feed exports
|
||||
<topics-feed-exports>` documentation. (:issue:`6183`)
|
||||
|
||||
- Added a missing note about backward-incompatible changes in
|
||||
:class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes.
|
||||
(:issue:`6060`, :issue:`6081`)
|
||||
|
||||
- Added a missing note about removing the deprecated
|
||||
``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes.
|
||||
(:issue:`6056`, :issue:`6061`)
|
||||
|
||||
- Other documentation improvements. (:issue:`6128`, :issue:`6144`,
|
||||
:issue:`6163`, :issue:`6190`, :issue:`6192`)
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added Python 3.12 to the CI configuration, re-enabled tests that were
|
||||
disabled when the pre-release support was added. (:issue:`5985`,
|
||||
:issue:`6083`, :issue:`6098`)
|
||||
|
||||
- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`)
|
||||
|
||||
|
||||
.. _release-2.11.0:
|
||||
|
||||
Scrapy 2.11.0 (2023-09-18)
|
||||
|
|
@ -32,8 +131,10 @@ Backward-incompatible changes
|
|||
:meth:`scrapy.crawler.Crawler.__init__` and before the settings are
|
||||
finalized and frozen. This change was needed to allow changing the settings
|
||||
in :meth:`scrapy.Spider.from_crawler`. If you want to access the final
|
||||
setting values in the spider code as early as possible you can do this in
|
||||
:meth:`~scrapy.Spider.start_requests`. (:issue:`6038`)
|
||||
setting values and the initialized :class:`~scrapy.crawler.Crawler`
|
||||
attributes in the spider code as early as possible you can do this in
|
||||
:meth:`~scrapy.Spider.start_requests` or in a handler of the
|
||||
:signal:`engine_started` signal. (:issue:`6038`)
|
||||
|
||||
- The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
|
||||
requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
|
||||
|
|
@ -60,6 +161,9 @@ Deprecation removals
|
|||
1.0.0, use :attr:`CrawlerRunner.spider_loader
|
||||
<scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
|
||||
|
||||
- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in
|
||||
Scrapy 2.6.0, has now been removed. (:issue:`6111`)
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -1155,6 +1259,9 @@ Deprecations
|
|||
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
|
||||
first to set the :class:`~scrapy.Spider` object.
|
||||
|
||||
- :func:`scrapy.utils.response.response_httprepr` is now deprecated.
|
||||
(:issue:`4972`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
|
@ -2869,6 +2976,38 @@ affect subclasses:
|
|||
|
||||
(:issue:`3884`)
|
||||
|
||||
.. _release-1.8.4:
|
||||
|
||||
Scrapy 1.8.4 (2024-02-14)
|
||||
-------------------------
|
||||
|
||||
**Security bug fixes:**
|
||||
|
||||
- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is
|
||||
now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`,
|
||||
which :class:`~scrapy.spiders.XMLFeedSpider` now uses.
|
||||
|
||||
To minimize the impact of this change on existing code,
|
||||
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
|
||||
the node namespace as a prefix in the node name, and big files with highly
|
||||
nested trees when using libxml2 2.7+.
|
||||
|
||||
Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information.
|
||||
|
||||
- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply
|
||||
to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security
|
||||
advisory`_ for more information.
|
||||
|
||||
- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the
|
||||
``scrapy.downloadermiddlewares.decompression`` module is discouraged and
|
||||
will trigger a warning.
|
||||
|
||||
- The ``Authorization`` header is now dropped on redirects to a different
|
||||
domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv
|
||||
|
||||
|
||||
.. _release-1.8.3:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
sphinx==5.0.2
|
||||
sphinx-hoverxref==1.1.1
|
||||
sphinx-notfound-page==0.8
|
||||
sphinx-rtd-theme==1.0.0
|
||||
sphinx==6.2.1
|
||||
sphinx-hoverxref==1.3.0
|
||||
sphinx-notfound-page==1.0.0
|
||||
sphinx-rtd-theme==2.0.0
|
||||
|
|
|
|||
|
|
@ -150,8 +150,7 @@ Access the crawler instance:
|
|||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def update_settings(self, settings):
|
||||
...
|
||||
def update_settings(self, settings): ...
|
||||
|
||||
Use a fallback component:
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY
|
|||
Default: ``1.0``
|
||||
|
||||
Average number of requests Scrapy should be sending in parallel to remote
|
||||
websites.
|
||||
websites. It must be higher than ``0.0``.
|
||||
|
||||
By default, AutoThrottle adjusts the delay to send a single
|
||||
concurrent request to each of the remote websites. Set this option to
|
||||
|
|
|
|||
|
|
@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see
|
|||
|
||||
See also: :ref:`topics-shell-inspect-response`.
|
||||
|
||||
|
||||
Open in browser
|
||||
===============
|
||||
|
||||
Sometimes you just want to see how a certain response looks in a browser, you
|
||||
can use the ``open_in_browser`` function for that. Here is an example of how
|
||||
you would use it:
|
||||
can use the :func:`~scrapy.utils.response.open_in_browser` function for that:
|
||||
|
||||
.. code-block:: python
|
||||
.. autofunction:: scrapy.utils.response.open_in_browser
|
||||
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
||||
def parse_details(self, response):
|
||||
if "item name" not in response.body:
|
||||
open_in_browser(response)
|
||||
|
||||
``open_in_browser`` will open a browser with the response received by Scrapy at
|
||||
that point, adjusting the `base tag`_ so that images and styles are displayed
|
||||
properly.
|
||||
|
||||
Logging
|
||||
=======
|
||||
|
|
@ -163,8 +153,6 @@ available in all future runs should they be necessary again:
|
|||
|
||||
For more information, check the :ref:`topics-logging` section.
|
||||
|
||||
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
|
||||
|
||||
.. _debug-vscode:
|
||||
|
||||
Visual Studio Code
|
||||
|
|
|
|||
|
|
@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which
|
|||
allows you to generate feeds with the scraped items, using multiple
|
||||
serialization formats and storage backends.
|
||||
|
||||
This page provides detailed documentation for all feed export features. If you
|
||||
are looking for a step-by-step guide, check out `Zyte’s export guides`_.
|
||||
|
||||
.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data
|
||||
|
||||
.. _topics-feed-format:
|
||||
|
||||
Serialization formats
|
||||
|
|
@ -385,7 +390,13 @@ Each plugin is a class that must implement the following methods:
|
|||
|
||||
.. method:: close(self)
|
||||
|
||||
Close the target file object.
|
||||
Clean up the plugin.
|
||||
|
||||
For example, you might want to close a file wrapper that you might have
|
||||
used to compress data written into the file received in the ``__init__``
|
||||
method.
|
||||
|
||||
.. warning:: Do not close the file from the ``__init__`` method.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
|
|
|||
|
|
@ -399,12 +399,7 @@ In code that receives an item, such as methods of :ref:`item pipelines
|
|||
<topics-spider-middleware>`, it is a good practice to use the
|
||||
:class:`~itemadapter.ItemAdapter` class and the
|
||||
:func:`~itemadapter.is_item` function to write code that works for
|
||||
any :ref:`supported item type <item-types>`:
|
||||
|
||||
.. autoclass:: itemadapter.ItemAdapter
|
||||
|
||||
.. autofunction:: itemadapter.is_item
|
||||
|
||||
any supported item type.
|
||||
|
||||
Other classes related to items
|
||||
==============================
|
||||
|
|
|
|||
|
|
@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
.. code-block:: python
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
from urllib.parse import urlparse
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
from scrapy.pipelines.files import FilesPipeline
|
||||
|
||||
|
||||
class MyFilesPipeline(FilesPipeline):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return "files/" + PurePosixPath(urlparse(request.url).path).name
|
||||
return "files/" + PurePosixPath(urlparse_cached(request).path).name
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
|
@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
.. code-block:: python
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
from urllib.parse import urlparse
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
|
||||
|
||||
class MyImagesPipeline(ImagesPipeline):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return "files/" + PurePosixPath(urlparse(request.url).path).name
|
||||
return "files/" + PurePosixPath(urlparse_cached(request).path).name
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
|
|
|||
|
|
@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
super proxy that you can attach your own proxies to.
|
||||
* use a highly distributed downloader that circumvents bans internally, so you
|
||||
can just focus on parsing clean pages. One example of such downloaders is
|
||||
`Zyte Smart Proxy Manager`_
|
||||
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
|
||||
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
|
||||
|
||||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
|
@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html
|
||||
|
|
|
|||
|
|
@ -193,18 +193,47 @@ Request objects
|
|||
:meth:`replace`.
|
||||
|
||||
.. attribute:: Request.meta
|
||||
:value: {}
|
||||
|
||||
A dict that contains arbitrary metadata for this request. This dict is
|
||||
empty for new Requests, and is usually populated by different Scrapy
|
||||
components (extensions, middlewares, etc). So the data contained in this
|
||||
dict depends on the extensions you have enabled.
|
||||
A dictionary of arbitrary metadata for the request.
|
||||
|
||||
See :ref:`topics-request-meta` for a list of special meta keys
|
||||
recognized by Scrapy.
|
||||
You may extend request metadata as you see fit.
|
||||
|
||||
This dict is :doc:`shallow copied <library/copy>` when the request is
|
||||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.meta`` attribute.
|
||||
Request metadata can also be accessed through the
|
||||
:attr:`~scrapy.http.Response.meta` attribute of a response.
|
||||
|
||||
To pass data from one spider callback to another, consider using
|
||||
:attr:`cb_kwargs` instead. However, request metadata may be the right
|
||||
choice in certain scenarios, such as to maintain some debugging data
|
||||
across all follow-up requests (e.g. the source URL).
|
||||
|
||||
A common use of request metadata is to define request-specific
|
||||
parameters for Scrapy components (extensions, middlewares, etc.). For
|
||||
example, if you set ``dont_retry`` to ``True``,
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never
|
||||
retry that request, even if it fails. See :ref:`topics-request-meta`.
|
||||
|
||||
You may also use request metadata in your custom Scrapy components, for
|
||||
example, to keep request state information relevant to your component.
|
||||
For example,
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the
|
||||
``retry_times`` metadata key to keep track of how many times a request
|
||||
has been retried so far.
|
||||
|
||||
Copying all the metadata of a previous request into a new, follow-up
|
||||
request in a spider callback is a bad practice, because request
|
||||
metadata may include metadata set by Scrapy components that is not
|
||||
meant to be copied into other requests. For example, copying the
|
||||
``retry_times`` metadata key into follow-up requests can lower the
|
||||
amount of retries allowed for those follow-up requests.
|
||||
|
||||
You should only copy all request metadata from one request to another
|
||||
if the new request is meant to replace the old request, as is often the
|
||||
case when returning a request from a :ref:`downloader middleware
|
||||
<topics-downloader-middleware>` method.
|
||||
|
||||
Also mind that the :meth:`copy` and :meth:`replace` request methods
|
||||
:doc:`shallow-copy <library/copy>` request metadata.
|
||||
|
||||
.. attribute:: Request.cb_kwargs
|
||||
|
||||
|
|
@ -440,60 +469,6 @@ import path.
|
|||
|
||||
.. autoclass:: scrapy.utils.request.RequestFingerprinter
|
||||
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.7
|
||||
|
||||
Default: ``'2.6'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'2.6'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy 2.6 and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'2.7'``
|
||||
|
||||
This implementation was introduced in Scrapy 2.7 to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'2.6'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the 2.6 request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidate the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'2.6'``).
|
||||
|
||||
|
||||
.. _2.6-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
|
|
@ -702,6 +677,7 @@ Those are:
|
|||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_warnsize`
|
||||
* :reqmeta:`download_timeout`
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
|
||||
|
|
@ -1328,3 +1304,13 @@ XmlResponse objects
|
|||
line. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
|
||||
|
||||
JsonResponse objects
|
||||
--------------------
|
||||
|
||||
.. class:: JsonResponse(url[, ...])
|
||||
|
||||
The :class:`JsonResponse` class is a subclass of :class:`TextResponse`
|
||||
that is used when the response has a `JSON MIME type
|
||||
<https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
|
||||
header.
|
||||
|
|
|
|||
|
|
@ -1032,10 +1032,8 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently,
|
|||
so performance-wise its uses are limited to situations that are not easily
|
||||
described with CSS selectors.
|
||||
|
||||
Parsel also simplifies adding your own XPath extensions.
|
||||
|
||||
.. autofunction:: parsel.xpathfuncs.set_xpathfunc
|
||||
|
||||
Parsel also simplifies adding your own XPath extensions with
|
||||
:func:`~parsel.xpathfuncs.set_xpathfunc`.
|
||||
|
||||
.. _topics-selectors-ref:
|
||||
|
||||
|
|
|
|||
|
|
@ -873,40 +873,42 @@ The amount of time (in secs) that the downloader will wait before timing out.
|
|||
Request.meta key.
|
||||
|
||||
.. setting:: DOWNLOAD_MAXSIZE
|
||||
.. reqmeta:: download_maxsize
|
||||
|
||||
DOWNLOAD_MAXSIZE
|
||||
----------------
|
||||
|
||||
Default: ``1073741824`` (1024MB)
|
||||
Default: ``1073741824`` (1 GiB)
|
||||
|
||||
The maximum response size (in bytes) that downloader will download.
|
||||
The maximum response body size (in bytes) allowed. Bigger responses are
|
||||
aborted and ignored.
|
||||
|
||||
If you want to disable it set to 0.
|
||||
This applies both before and after compression. If decompressing a response
|
||||
body would exceed this limit, decompression is aborted and the response is
|
||||
ignored.
|
||||
|
||||
.. reqmeta:: download_maxsize
|
||||
Use ``0`` to disable this limit.
|
||||
|
||||
.. note::
|
||||
|
||||
This size can be set per spider using :attr:`download_maxsize`
|
||||
spider attribute and per-request using :reqmeta:`download_maxsize`
|
||||
Request.meta key.
|
||||
This limit can be set per spider using the :attr:`download_maxsize` spider
|
||||
attribute and per request using the :reqmeta:`download_maxsize` Request.meta
|
||||
key.
|
||||
|
||||
.. setting:: DOWNLOAD_WARNSIZE
|
||||
.. reqmeta:: download_warnsize
|
||||
|
||||
DOWNLOAD_WARNSIZE
|
||||
-----------------
|
||||
|
||||
Default: ``33554432`` (32MB)
|
||||
Default: ``33554432`` (32 MiB)
|
||||
|
||||
The response size (in bytes) that downloader will start to warn.
|
||||
If the size of a response exceeds this value, before or after compression, a
|
||||
warning will be logged about it.
|
||||
|
||||
If you want to disable it set to 0.
|
||||
Use ``0`` to disable this limit.
|
||||
|
||||
.. note::
|
||||
|
||||
This size can be set per spider using :attr:`download_warnsize`
|
||||
spider attribute and per-request using :reqmeta:`download_warnsize`
|
||||
Request.meta key.
|
||||
This limit can be set per spider using the :attr:`download_warnsize` spider
|
||||
attribute and per request using the :reqmeta:`download_warnsize` Request.meta
|
||||
key.
|
||||
|
||||
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
||||
|
|
|
|||
|
|
@ -142,8 +142,14 @@ scrapy.Spider
|
|||
method, which is handy if you want to modify them based on
|
||||
arguments. As a consequence, these settings aren't the final values
|
||||
as they can be modified later by e.g. :ref:`add-ons
|
||||
<topics-addons>`. The final settings are available in the
|
||||
:meth:`start_requests` method and later.
|
||||
<topics-addons>`. For the same reason, most of the
|
||||
:class:`~scrapy.crawler.Crawler` attributes aren't initialized at
|
||||
this point.
|
||||
|
||||
The final settings and the initialized
|
||||
:class:`~scrapy.crawler.Crawler` attributes are available in the
|
||||
:meth:`start_requests` method, handlers of the
|
||||
:signal:`engine_started` signal and later.
|
||||
|
||||
:param crawler: crawler to which the spider will be bound
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` instance
|
||||
|
|
|
|||
19
pylintrc
19
pylintrc
|
|
@ -4,21 +4,14 @@ jobs=1 # >1 hides results
|
|||
|
||||
[MESSAGES CONTROL]
|
||||
disable=abstract-method,
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
arguments-renamed,
|
||||
attribute-defined-outside-init,
|
||||
bad-classmethod-argument,
|
||||
bad-mcs-classmethod-argument,
|
||||
bare-except,
|
||||
broad-except,
|
||||
broad-exception-raised,
|
||||
c-extension-no-member,
|
||||
catching-non-exception,
|
||||
cell-var-from-loop,
|
||||
comparison-with-callable,
|
||||
consider-using-dict-items,
|
||||
consider-using-in,
|
||||
consider-using-with,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
|
|
@ -32,7 +25,6 @@ disable=abstract-method,
|
|||
implicit-str-concat,
|
||||
import-error,
|
||||
import-outside-toplevel,
|
||||
import-self,
|
||||
inconsistent-return-statements,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
|
|
@ -44,7 +36,6 @@ disable=abstract-method,
|
|||
logging-fstring-interpolation,
|
||||
logging-not-lazy,
|
||||
lost-exception,
|
||||
method-hidden,
|
||||
missing-docstring,
|
||||
no-else-raise,
|
||||
no-else-return,
|
||||
|
|
@ -52,7 +43,7 @@ disable=abstract-method,
|
|||
no-method-argument,
|
||||
no-name-in-module,
|
||||
no-self-argument,
|
||||
no-value-for-parameter,
|
||||
no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268
|
||||
not-callable,
|
||||
pointless-exception-statement,
|
||||
pointless-statement,
|
||||
|
|
@ -77,23 +68,15 @@ disable=abstract-method,
|
|||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
unbalanced-tuple-unpacking,
|
||||
undefined-variable,
|
||||
undefined-loop-variable,
|
||||
unexpected-special-method-signature,
|
||||
unnecessary-comprehension,
|
||||
unnecessary-dunder-call,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unsubscriptable-object,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-private-member,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
use-dict-literal,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
useless-super-delegation,
|
||||
wildcard-import,
|
||||
wrong-import-position
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.11.0
|
||||
2.11.1
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
|
@ -32,9 +32,7 @@ class AddonManager:
|
|||
for clspath in build_component_list(settings["ADDONS"]):
|
||||
try:
|
||||
addoncls = load_object(clspath)
|
||||
addon = create_instance(
|
||||
addoncls, settings=settings, crawler=self.crawler
|
||||
)
|
||||
addon = build_from_crawler(addoncls, self.crawler)
|
||||
addon.update_settings(settings)
|
||||
self.addons.append(addon)
|
||||
except NotConfigured as e:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Base class for Scrapy commands
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import functools
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
|
|
@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
return scraped_data
|
||||
|
||||
def _get_callback(self, *, spider, opts, response=None):
|
||||
cb = None
|
||||
if response:
|
||||
cb = response.meta["_callback"]
|
||||
if not cb:
|
||||
if opts.callback:
|
||||
cb = opts.callback
|
||||
elif response and opts.rules and self.first_response == response:
|
||||
cb = self.get_callback_from_rules(spider, response)
|
||||
if not cb:
|
||||
raise ValueError(
|
||||
f"Cannot find a rule that matches {response.url!r} in spider: "
|
||||
f"{spider.name}"
|
||||
)
|
||||
else:
|
||||
cb = "parse"
|
||||
|
||||
if not callable(cb):
|
||||
cb_method = getattr(spider, cb, None)
|
||||
if callable(cb_method):
|
||||
cb = cb_method
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Cannot find callback {cb!r} in spider: {spider.name}"
|
||||
)
|
||||
return cb
|
||||
|
||||
def prepare_request(self, spider, request, opts):
|
||||
def callback(response, **cb_kwargs):
|
||||
# memorize first request
|
||||
if not self.first_response:
|
||||
self.first_response = response
|
||||
|
||||
# determine real callback
|
||||
cb = response.meta["_callback"]
|
||||
if not cb:
|
||||
if opts.callback:
|
||||
cb = opts.callback
|
||||
elif opts.rules and self.first_response == response:
|
||||
cb = self.get_callback_from_rules(spider, response)
|
||||
|
||||
if not cb:
|
||||
logger.error(
|
||||
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
|
||||
{"url": response.url, "spider": spider.name},
|
||||
)
|
||||
return
|
||||
else:
|
||||
cb = "parse"
|
||||
|
||||
if not callable(cb):
|
||||
cb_method = getattr(spider, cb, None)
|
||||
if callable(cb_method):
|
||||
cb = cb_method
|
||||
else:
|
||||
logger.error(
|
||||
"Cannot find callback %(callback)r in spider: %(spider)s",
|
||||
{"callback": cb, "spider": spider.name},
|
||||
)
|
||||
return
|
||||
cb = self._get_callback(spider=spider, opts=opts, response=response)
|
||||
|
||||
# parse items and requests
|
||||
depth = response.meta["_depth"]
|
||||
|
|
@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
request.meta["_depth"] = 1
|
||||
request.meta["_callback"] = request.callback
|
||||
if not request.callback and not opts.rules:
|
||||
cb = self._get_callback(spider=spider, opts=opts)
|
||||
functools.update_wrapper(callback, cb)
|
||||
request.callback = callback
|
||||
return request
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Scrapy Shell
|
|||
|
||||
See documentation in docs/topics/shell.rst
|
||||
"""
|
||||
|
||||
from argparse import Namespace
|
||||
from threading import Thread
|
||||
from typing import List, Type
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from scrapy.core.downloader.tls import (
|
|||
openssl_methods,
|
||||
)
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from twisted.internet._sslverify import ClientTLSOptions
|
||||
|
|
@ -165,18 +165,16 @@ def load_context_factory_from_settings(settings, crawler):
|
|||
context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
|
||||
# try method-aware context factory
|
||||
try:
|
||||
context_factory = create_instance(
|
||||
objcls=context_factory_cls,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
context_factory = build_from_crawler(
|
||||
context_factory_cls,
|
||||
crawler,
|
||||
method=ssl_method,
|
||||
)
|
||||
except TypeError:
|
||||
# use context factory defaults
|
||||
context_factory = create_instance(
|
||||
objcls=context_factory_cls,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
context_factory = build_from_crawler(
|
||||
context_factory_cls,
|
||||
crawler,
|
||||
)
|
||||
msg = (
|
||||
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred
|
|||
from scrapy import Request, Spider, signals
|
||||
from scrapy.exceptions import NotConfigured, NotSupported
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.python import without_none_values
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
|
|||
class DownloadHandlers:
|
||||
def __init__(self, crawler: "Crawler"):
|
||||
self._crawler: "Crawler" = crawler
|
||||
self._schemes: Dict[
|
||||
str, Union[str, Callable]
|
||||
] = {} # stores acceptable schemes on instancing
|
||||
self._schemes: Dict[str, Union[str, Callable]] = (
|
||||
{}
|
||||
) # stores acceptable schemes on instancing
|
||||
self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes
|
||||
self._notconfigured: Dict[str, str] = {} # remembers failed handlers
|
||||
handlers: Dict[str, Union[str, Callable]] = without_none_values(
|
||||
|
|
@ -55,10 +55,9 @@ class DownloadHandlers:
|
|||
dhcls = load_object(path)
|
||||
if skip_lazy and getattr(dhcls, "lazy", True):
|
||||
return None
|
||||
dh = create_instance(
|
||||
objcls=dhcls,
|
||||
settings=self._crawler.settings,
|
||||
crawler=self._crawler,
|
||||
dh = build_from_crawler(
|
||||
dhcls,
|
||||
self._crawler,
|
||||
)
|
||||
except NotConfigured as ex:
|
||||
self._notconfigured[scheme] = str(ex)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Download handlers for http and https schemes
|
||||
"""
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
|
|
@ -30,10 +31,9 @@ class HTTP10DownloadHandler:
|
|||
|
||||
host, port = to_unicode(factory.host), factory.port
|
||||
if factory.scheme == b"https":
|
||||
client_context_factory = create_instance(
|
||||
objcls=self.ClientContextFactory,
|
||||
settings=self._settings,
|
||||
crawler=self._crawler,
|
||||
client_context_factory = build_from_crawler(
|
||||
self.ClientContextFactory,
|
||||
self._crawler,
|
||||
)
|
||||
return reactor.connectSSL(host, port, factory, client_context_factory)
|
||||
return reactor.connectTCP(host, port, factory)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
|
||||
|
||||
class S3DownloadHandler:
|
||||
|
|
@ -50,10 +50,9 @@ class S3DownloadHandler:
|
|||
)
|
||||
)
|
||||
|
||||
_http_handler = create_instance(
|
||||
objcls=httpdownloadhandler,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
_http_handler = build_from_crawler(
|
||||
httpdownloadhandler,
|
||||
crawler,
|
||||
)
|
||||
self._download_http = _http_handler.download_request
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Generator, List, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
|
|||
For more information see docs/topics/architecture.rst
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
from time import time
|
||||
from typing import (
|
||||
|
|
@ -34,7 +35,7 @@ from scrapy.settings import BaseSettings, Settings
|
|||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.reactor import CallLaterOnce
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -358,9 +359,7 @@ class ExecutionEngine:
|
|||
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
|
||||
logger.info("Spider opened", extra={"spider": spider})
|
||||
nextcall = CallLaterOnce(self._next_request)
|
||||
scheduler = create_instance(
|
||||
self.scheduler_cls, settings=None, crawler=self.crawler
|
||||
)
|
||||
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
|
||||
start_requests = yield self.scraper.spidermw.process_start_requests(
|
||||
start_requests, spider
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import logging
|
|||
from enum import Enum
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from h2.errors import ErrorCodes
|
||||
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
|
||||
|
|
@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailed
|
|||
from scrapy.http import Request
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol
|
||||
|
|
@ -111,17 +111,17 @@ class Stream:
|
|||
# Metadata of an HTTP/2 connection stream
|
||||
# initialized when stream is instantiated
|
||||
self.metadata: Dict = {
|
||||
"request_content_length": 0
|
||||
if self._request.body is None
|
||||
else len(self._request.body),
|
||||
"request_content_length": (
|
||||
0 if self._request.body is None else len(self._request.body)
|
||||
),
|
||||
# Flag to keep track whether the stream has initiated the request
|
||||
"request_sent": False,
|
||||
# Flag to track whether we have logged about exceeding download warnsize
|
||||
"reached_warnsize": False,
|
||||
# Each time we send a data frame, we will decrease value by the amount send.
|
||||
"remaining_content_length": 0
|
||||
if self._request.body is None
|
||||
else len(self._request.body),
|
||||
"remaining_content_length": (
|
||||
0 if self._request.body is None else len(self._request.body)
|
||||
),
|
||||
# Flag to keep track whether client (self) have closed this stream
|
||||
"stream_closed_local": False,
|
||||
# Flag to keep track whether the server has closed the stream
|
||||
|
|
@ -185,7 +185,7 @@ class Stream:
|
|||
|
||||
def check_request_url(self) -> bool:
|
||||
# Make sure that we are sending the request to the correct URL
|
||||
url = urlparse(self._request.url)
|
||||
url = urlparse_cached(self._request)
|
||||
return (
|
||||
url.netloc == str(self._protocol.metadata["uri"].host, "utf-8")
|
||||
or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8")
|
||||
|
|
@ -194,7 +194,7 @@ class Stream:
|
|||
)
|
||||
|
||||
def _get_request_headers(self) -> List[Tuple[str, str]]:
|
||||
url = urlparse(self._request.url)
|
||||
url = urlparse_cached(self._request)
|
||||
|
||||
path = url.path
|
||||
if url.query:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from scrapy.http.request import Request
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
|
|
@ -202,7 +202,7 @@ class Scheduler(BaseScheduler):
|
|||
"""
|
||||
dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"])
|
||||
return cls(
|
||||
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
|
||||
dupefilter=build_from_crawler(dupefilter_cls, crawler),
|
||||
jobdir=job_dir(crawler.settings),
|
||||
dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]),
|
||||
mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]),
|
||||
|
|
@ -322,10 +322,9 @@ class Scheduler(BaseScheduler):
|
|||
|
||||
def _mq(self):
|
||||
"""Create a new priority queue instance, with in-memory storage"""
|
||||
return create_instance(
|
||||
return build_from_crawler(
|
||||
self.pqclass,
|
||||
settings=None,
|
||||
crawler=self.crawler,
|
||||
self.crawler,
|
||||
downstream_queue_cls=self.mqclass,
|
||||
key="",
|
||||
)
|
||||
|
|
@ -334,10 +333,9 @@ class Scheduler(BaseScheduler):
|
|||
"""Create a new priority queue instance, with disk storage"""
|
||||
assert self.dqdir
|
||||
state = self._read_dqs_state(self.dqdir)
|
||||
q = create_instance(
|
||||
q = build_from_crawler(
|
||||
self.pqclass,
|
||||
settings=None,
|
||||
crawler=self.crawler,
|
||||
self.crawler,
|
||||
downstream_queue_cls=self.dqclass,
|
||||
key=self.dqdir,
|
||||
startprios=state,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""This module implements the Scraper component which parses responses and
|
||||
extracts information from them"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Spider Middleware manager
|
|||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
import logging
|
||||
from inspect import isasyncgenfunction, iscoroutine
|
||||
from itertools import islice
|
||||
|
|
@ -103,8 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
) -> Union[Generator, AsyncGenerator]:
|
||||
def process_sync(iterable: Iterable) -> Generator:
|
||||
try:
|
||||
for r in iterable:
|
||||
yield r
|
||||
yield from iterable
|
||||
except Exception as ex:
|
||||
exception_result = self._process_spider_exception(
|
||||
response, spider, Failure(ex), exception_processor_index
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ from scrapy.utils.log import (
|
|||
log_reactor_info,
|
||||
log_scrapy_info,
|
||||
)
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
|
||||
from scrapy.utils.reactor import (
|
||||
install_reactor,
|
||||
|
|
@ -109,10 +109,9 @@ class Crawler:
|
|||
lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
self.request_fingerprinter = create_instance(
|
||||
self.request_fingerprinter = build_from_crawler(
|
||||
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
|
||||
settings=self.settings,
|
||||
crawler=self,
|
||||
self,
|
||||
)
|
||||
|
||||
reactor_class: str = self.settings["TWISTED_REACTOR"]
|
||||
|
|
@ -179,6 +178,48 @@ class Crawler:
|
|||
assert self.engine
|
||||
yield maybeDeferred(self.engine.stop)
|
||||
|
||||
@staticmethod
|
||||
def _get_component(component_class, components):
|
||||
for component in components:
|
||||
if isinstance(component, component_class):
|
||||
return component
|
||||
return None
|
||||
|
||||
def get_addon(self, cls):
|
||||
return self._get_component(cls, self.addons.addons)
|
||||
|
||||
def get_downloader_middleware(self, cls):
|
||||
if not self.engine:
|
||||
raise RuntimeError(
|
||||
"Crawler.get_downloader_middleware() can only be called after "
|
||||
"the crawl engine has been created."
|
||||
)
|
||||
return self._get_component(cls, self.engine.downloader.middleware.middlewares)
|
||||
|
||||
def get_extension(self, cls):
|
||||
if not self.extensions:
|
||||
raise RuntimeError(
|
||||
"Crawler.get_extension() can only be called after the "
|
||||
"extension manager has been created."
|
||||
)
|
||||
return self._get_component(cls, self.extensions.middlewares)
|
||||
|
||||
def get_item_pipeline(self, cls):
|
||||
if not self.engine:
|
||||
raise RuntimeError(
|
||||
"Crawler.get_item_pipeline() can only be called after the "
|
||||
"crawl engine has been created."
|
||||
)
|
||||
return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
|
||||
|
||||
def get_spider_middleware(self, cls):
|
||||
if not self.engine:
|
||||
raise RuntimeError(
|
||||
"Crawler.get_spider_middleware() can only be called after the "
|
||||
"crawl engine has been created."
|
||||
)
|
||||
return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
|
||||
|
||||
|
||||
class CrawlerRunner:
|
||||
"""
|
||||
|
|
@ -404,7 +445,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
d.addBoth(self._stop_reactor)
|
||||
|
||||
resolver_class = load_object(self.settings["DNS_RESOLVER"])
|
||||
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
|
||||
resolver = build_from_crawler(resolver_class, self, reactor=reactor)
|
||||
resolver.install_on_reactor()
|
||||
tp = reactor.getThreadPool()
|
||||
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ DefaultHeaders downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Iterable, Tuple, Union
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Download timeout middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
|
|
|||
|
|
@ -1,50 +1,93 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zlib
|
||||
import warnings
|
||||
from itertools import chain
|
||||
from logging import getLogger
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.utils._compression import (
|
||||
_DecompressionMaxSizeExceeded,
|
||||
_inflate,
|
||||
_unbrotli,
|
||||
_unzstd,
|
||||
)
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.gz import gunzip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
|
||||
|
||||
try:
|
||||
import brotli
|
||||
|
||||
ACCEPTED_ENCODINGS.append(b"br")
|
||||
import brotli # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPTED_ENCODINGS.append(b"br")
|
||||
|
||||
try:
|
||||
import zstandard
|
||||
|
||||
ACCEPTED_ENCODINGS.append(b"zstd")
|
||||
import zstandard # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPTED_ENCODINGS.append(b"zstd")
|
||||
|
||||
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites"""
|
||||
|
||||
def __init__(self, stats: Optional[StatsCollector] = None):
|
||||
self.stats = stats
|
||||
def __init__(
|
||||
self,
|
||||
stats: Optional[StatsCollector] = None,
|
||||
*,
|
||||
crawler: Optional[Crawler] = None,
|
||||
):
|
||||
if not crawler:
|
||||
self.stats = stats
|
||||
self._max_size = 1073741824
|
||||
self._warn_size = 33554432
|
||||
return
|
||||
self.stats = crawler.stats
|
||||
self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
|
||||
self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
|
||||
crawler.signals.connect(self.open_spider, signals.spider_opened)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
if not crawler.settings.getbool("COMPRESSION_ENABLED"):
|
||||
raise NotConfigured
|
||||
return cls(stats=crawler.stats)
|
||||
try:
|
||||
return cls(crawler=crawler)
|
||||
except TypeError:
|
||||
warnings.warn(
|
||||
"HttpCompressionMiddleware subclasses must either modify "
|
||||
"their '__init__' method to support a 'crawler' parameter or "
|
||||
"reimplement their 'from_crawler' method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
mw = cls()
|
||||
mw.stats = crawler.stats
|
||||
mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
|
||||
mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
|
||||
crawler.signals.connect(mw.open_spider, signals.spider_opened)
|
||||
return mw
|
||||
|
||||
def open_spider(self, spider):
|
||||
if hasattr(spider, "download_maxsize"):
|
||||
self._max_size = spider.download_maxsize
|
||||
if hasattr(spider, "download_warnsize"):
|
||||
self._warn_size = spider.download_warnsize
|
||||
|
||||
def process_request(
|
||||
self, request: Request, spider: Spider
|
||||
|
|
@ -60,8 +103,26 @@ class HttpCompressionMiddleware:
|
|||
if isinstance(response, Response):
|
||||
content_encoding = response.headers.getlist("Content-Encoding")
|
||||
if content_encoding:
|
||||
encoding = content_encoding.pop()
|
||||
decoded_body = self._decode(response.body, encoding.lower())
|
||||
max_size = request.meta.get("download_maxsize", self._max_size)
|
||||
warn_size = request.meta.get("download_warnsize", self._warn_size)
|
||||
try:
|
||||
decoded_body, content_encoding = self._handle_encoding(
|
||||
response.body, content_encoding, max_size
|
||||
)
|
||||
except _DecompressionMaxSizeExceeded:
|
||||
raise IgnoreRequest(
|
||||
f"Ignored response {response} because its body "
|
||||
f"({len(response.body)} B compressed) exceeded "
|
||||
f"DOWNLOAD_MAXSIZE ({max_size} B) during "
|
||||
f"decompression."
|
||||
)
|
||||
if len(response.body) < warn_size <= len(decoded_body):
|
||||
logger.warning(
|
||||
f"{response} body size after decompression "
|
||||
f"({len(decoded_body)} B) is larger than the "
|
||||
f"download warning size ({warn_size} B)."
|
||||
)
|
||||
response.headers["Content-Encoding"] = content_encoding
|
||||
if self.stats:
|
||||
self.stats.inc_value(
|
||||
"httpcompression/response_bytes",
|
||||
|
|
@ -74,7 +135,7 @@ class HttpCompressionMiddleware:
|
|||
respcls = responsetypes.from_args(
|
||||
headers=response.headers, url=response.url, body=decoded_body
|
||||
)
|
||||
kwargs: Dict[str, Any] = dict(body=decoded_body)
|
||||
kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body}
|
||||
if issubclass(respcls, TextResponse):
|
||||
# force recalculating the encoding until we make sure the
|
||||
# responsetypes guessing is reliable
|
||||
|
|
@ -85,25 +146,35 @@ class HttpCompressionMiddleware:
|
|||
|
||||
return response
|
||||
|
||||
def _decode(self, body: bytes, encoding: bytes) -> bytes:
|
||||
if encoding == b"gzip" or encoding == b"x-gzip":
|
||||
body = gunzip(body)
|
||||
def _handle_encoding(self, body, content_encoding, max_size):
|
||||
to_decode, to_keep = self._split_encodings(content_encoding)
|
||||
for encoding in to_decode:
|
||||
body = self._decode(body, encoding, max_size)
|
||||
return body, to_keep
|
||||
|
||||
def _split_encodings(self, content_encoding):
|
||||
to_keep = [
|
||||
encoding.strip().lower()
|
||||
for encoding in chain.from_iterable(
|
||||
encodings.split(b",") for encodings in content_encoding
|
||||
)
|
||||
]
|
||||
to_decode = []
|
||||
while to_keep:
|
||||
encoding = to_keep.pop()
|
||||
if encoding not in ACCEPTED_ENCODINGS:
|
||||
to_keep.append(encoding)
|
||||
return to_decode, to_keep
|
||||
to_decode.append(encoding)
|
||||
return to_decode, to_keep
|
||||
|
||||
def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
|
||||
if encoding in {b"gzip", b"x-gzip"}:
|
||||
return gunzip(body, max_size=max_size)
|
||||
if encoding == b"deflate":
|
||||
try:
|
||||
body = zlib.decompress(body)
|
||||
except zlib.error:
|
||||
# ugly hack to work with raw deflate content that may
|
||||
# be sent by microsoft servers. For more information, see:
|
||||
# http://carsten.codimi.de/gzip.yaws/
|
||||
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
|
||||
# http://www.gzip.org/zlib/zlib_faq.html#faq38
|
||||
body = zlib.decompress(body, -15)
|
||||
return _inflate(body, max_size=max_size)
|
||||
if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
|
||||
body = brotli.decompress(body)
|
||||
return _unbrotli(body, max_size=max_size)
|
||||
if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
|
||||
# Using its streaming API since its simple API could handle only cases
|
||||
# where there is content size data embedded in the frame
|
||||
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
|
||||
body = reader.read()
|
||||
return _unzstd(body, max_size=max_size)
|
||||
return body
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, List, Union, cast
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -30,11 +30,17 @@ def _build_redirect_request(
|
|||
cls=None,
|
||||
cookies=None,
|
||||
)
|
||||
if "Cookie" in redirect_request.headers:
|
||||
has_cookie_header = "Cookie" in redirect_request.headers
|
||||
has_authorization_header = "Authorization" in redirect_request.headers
|
||||
if has_cookie_header or has_authorization_header:
|
||||
source_request_netloc = urlparse_cached(source_request).netloc
|
||||
redirect_request_netloc = urlparse_cached(redirect_request).netloc
|
||||
if source_request_netloc != redirect_request_netloc:
|
||||
del redirect_request.headers["Cookie"]
|
||||
if has_cookie_header:
|
||||
del redirect_request.headers["Cookie"]
|
||||
# https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name
|
||||
if has_authorization_header:
|
||||
del redirect_request.headers["Authorization"]
|
||||
return redirect_request
|
||||
|
||||
|
||||
|
|
@ -120,7 +126,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
assert response.headers["Location"] is not None
|
||||
location = safe_url_string(response.headers["Location"])
|
||||
if response.headers["Location"].startswith(b"//"):
|
||||
request_scheme = urlparse(request.url).scheme
|
||||
request_scheme = urlparse_cached(request).scheme
|
||||
location = request_scheme + "://" + location.lstrip("/")
|
||||
|
||||
redirected_url = urljoin(request.url, location)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
|
|||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non failed) pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Scrapy core exceptions
|
|||
These exceptions are documented in docs/topics/exceptions.rst. Please don't add
|
||||
new exceptions here without documenting them there.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Internal
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ The Extension Manager
|
|||
|
||||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Extension for collecting core stats like items scraped and start/finish times
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from scrapy import signals
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
|
|||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.python import without_none_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -371,7 +371,7 @@ class FeedSlot:
|
|||
self._exporting = True
|
||||
|
||||
def _get_instance(self, objcls, *args, **kwargs):
|
||||
return create_instance(objcls, self.settings, self.crawler, *args, **kwargs)
|
||||
return build_from_crawler(objcls, self.crawler, *args, **kwargs)
|
||||
|
||||
def _get_exporter(self, file, format, *args, **kwargs):
|
||||
return self._get_instance(self.exporters[format], file, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
|
||||
class LogStats:
|
||||
"""Log basic scraping stats periodically"""
|
||||
"""Log basic scraping stats periodically like:
|
||||
* RPM - Requests per Minute
|
||||
* IPM - Items per Minute
|
||||
"""
|
||||
|
||||
def __init__(self, stats, interval=60.0):
|
||||
self.stats = stats
|
||||
|
|
@ -35,24 +38,45 @@ class LogStats:
|
|||
self.task.start(self.interval)
|
||||
|
||||
def log(self, spider):
|
||||
items = self.stats.get_value("item_scraped_count", 0)
|
||||
pages = self.stats.get_value("response_received_count", 0)
|
||||
irate = (items - self.itemsprev) * self.multiplier
|
||||
prate = (pages - self.pagesprev) * self.multiplier
|
||||
self.pagesprev, self.itemsprev = pages, items
|
||||
self.calculate_stats()
|
||||
|
||||
msg = (
|
||||
"Crawled %(pages)d pages (at %(pagerate)d pages/min), "
|
||||
"scraped %(items)d items (at %(itemrate)d items/min)"
|
||||
)
|
||||
log_args = {
|
||||
"pages": pages,
|
||||
"pagerate": prate,
|
||||
"items": items,
|
||||
"itemrate": irate,
|
||||
"pages": self.pages,
|
||||
"pagerate": self.prate,
|
||||
"items": self.items,
|
||||
"itemrate": self.irate,
|
||||
}
|
||||
logger.info(msg, log_args, extra={"spider": spider})
|
||||
|
||||
def calculate_stats(self):
|
||||
self.items = self.stats.get_value("item_scraped_count", 0)
|
||||
self.pages = self.stats.get_value("response_received_count", 0)
|
||||
self.irate = (self.items - self.itemsprev) * self.multiplier
|
||||
self.prate = (self.pages - self.pagesprev) * self.multiplier
|
||||
self.pagesprev, self.itemsprev = self.pages, self.items
|
||||
|
||||
def spider_closed(self, spider, reason):
|
||||
if self.task and self.task.running:
|
||||
self.task.stop()
|
||||
|
||||
rpm_final, ipm_final = self.calculate_final_stats(spider)
|
||||
self.stats.set_value("responses_per_minute", rpm_final)
|
||||
self.stats.set_value("items_per_minute", ipm_final)
|
||||
|
||||
def calculate_final_stats(self, spider):
|
||||
start_time = self.stats.get_value("start_time")
|
||||
finished_time = self.stats.get_value("finished_time")
|
||||
|
||||
if not start_time or not finished_time:
|
||||
return None, None
|
||||
|
||||
mins_elapsed = (finished_time - start_time).seconds / 60
|
||||
|
||||
items = self.stats.get_value("item_scraped_count", 0)
|
||||
pages = self.stats.get_value("response_received_count", 0)
|
||||
|
||||
return (pages / mins_elapsed), (items / mins_elapsed)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ MemoryUsage extension
|
|||
|
||||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import sys
|
||||
|
|
@ -128,9 +129,9 @@ class MemoryUsage:
|
|||
def _send_report(self, rcpts, subject):
|
||||
"""send notification mail with some additional useful info"""
|
||||
stats = self.crawler.stats
|
||||
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
|
||||
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
|
||||
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
|
||||
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
|
||||
s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
|
||||
s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
|
||||
|
||||
s += (
|
||||
"ENGINE STATUS ------------------------------------------------------- \r\n"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Extension for processing data before they are exported to feeds.
|
||||
"""
|
||||
|
||||
from bz2 import BZ2File
|
||||
from gzip import GzipFile
|
||||
from io import IOBase
|
||||
|
|
@ -42,7 +43,6 @@ class GzipPlugin:
|
|||
|
||||
def close(self) -> None:
|
||||
self.gzipfile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class Bz2Plugin:
|
||||
|
|
@ -69,7 +69,6 @@ class Bz2Plugin:
|
|||
|
||||
def close(self) -> None:
|
||||
self.bz2file.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class LZMAPlugin:
|
||||
|
|
@ -111,7 +110,6 @@ class LZMAPlugin:
|
|||
|
||||
def close(self) -> None:
|
||||
self.lzmafile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
|
||||
|
|
|
|||
|
|
@ -16,6 +16,11 @@ class AutoThrottle:
|
|||
self.target_concurrency = crawler.settings.getfloat(
|
||||
"AUTOTHROTTLE_TARGET_CONCURRENCY"
|
||||
)
|
||||
if self.target_concurrency <= 0.0:
|
||||
raise NotConfigured(
|
||||
f"AUTOTHROTTLE_TARGET_CONCURRENCY "
|
||||
f"({self.target_concurrency!r}) must be higher than 0."
|
||||
)
|
||||
crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
|
||||
crawler.signals.connect(
|
||||
self._response_downloaded, signal=signals.response_downloaded
|
||||
|
|
|
|||
|
|
@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest
|
|||
from scrapy.http.request.rpc import XmlRpcRequest
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from scrapy.http.response.json import JsonResponse
|
||||
from scrapy.http.response.text import TextResponse
|
||||
from scrapy.http.response.xml import XmlResponse
|
||||
|
|
|
|||
|
|
@ -113,7 +113,9 @@ class Headers(CaselessDict):
|
|||
return ((k, self.getlist(k)) for k in self.keys())
|
||||
|
||||
def values(self) -> List[Optional[bytes]]: # type: ignore[override]
|
||||
return [self[k] for k in self.keys()]
|
||||
return [
|
||||
self[k] for k in self.keys() # pylint: disable=consider-using-dict-items
|
||||
]
|
||||
|
||||
def to_string(self) -> bytes:
|
||||
# cast() can be removed if the headers_dict_to_raw() hint is improved
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ requests in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
|
|
@ -187,12 +188,10 @@ class Request(object_ref):
|
|||
@overload
|
||||
def replace(
|
||||
self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
|
||||
) -> RequestTypeVar:
|
||||
...
|
||||
) -> RequestTypeVar: ...
|
||||
|
||||
@overload
|
||||
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
|
||||
...
|
||||
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
|
||||
|
||||
def replace(
|
||||
self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
|
||||
|
|
@ -252,12 +251,16 @@ class Request(object_ref):
|
|||
"""
|
||||
d = {
|
||||
"url": self.url, # urls are safe (safe_string_url)
|
||||
"callback": _find_method(spider, self.callback)
|
||||
if callable(self.callback)
|
||||
else self.callback,
|
||||
"errback": _find_method(spider, self.errback)
|
||||
if callable(self.errback)
|
||||
else self.errback,
|
||||
"callback": (
|
||||
_find_method(spider, self.callback)
|
||||
if callable(self.callback)
|
||||
else self.callback
|
||||
),
|
||||
"errback": (
|
||||
_find_method(spider, self.errback)
|
||||
if callable(self.errback)
|
||||
else self.errback
|
||||
),
|
||||
"headers": dict(self.headers),
|
||||
}
|
||||
for attr in self.attributes:
|
||||
|
|
|
|||
|
|
@ -4,12 +4,17 @@ This module implements the XmlRpcRequest class which is a more convenient class
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import xmlrpc.client as xmlrpclib
|
||||
from typing import Any, Optional
|
||||
|
||||
import defusedxml.xmlrpc
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
||||
defusedxml.xmlrpc.monkey_patch()
|
||||
|
||||
DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ responses in Scrapy.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ipaddress import IPv4Address, IPv6Address
|
||||
|
|
@ -147,12 +148,10 @@ class Response(object_ref):
|
|||
@overload
|
||||
def replace(
|
||||
self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
|
||||
) -> ResponseTypeVar:
|
||||
...
|
||||
) -> ResponseTypeVar: ...
|
||||
|
||||
@overload
|
||||
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
|
||||
...
|
||||
def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
|
||||
|
||||
def replace(
|
||||
self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
"""
|
||||
This module implements the JsonResponse class that is used when the response
|
||||
has a JSON MIME type in its Content-Type header.
|
||||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class JsonResponse(TextResponse):
|
||||
pass
|
||||
|
|
@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class.
|
|||
|
||||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
|
|||
For actual link extractors implementation see scrapy.linkextractors, or
|
||||
its documentation in: docs/topics/link-extractors.rst
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ This package contains a collection of Link Extractors.
|
|||
|
||||
For more info see docs/topics/link-extractors.rst
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# common file extensions that are not followed if they occur in links
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Link extractor based on lxml.html
|
||||
"""
|
||||
|
||||
import logging
|
||||
import operator
|
||||
from functools import partial
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Item Loader
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
|
||||
import itemloaders
|
||||
|
||||
from scrapy.item import Item
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Mail sending helpers
|
|||
|
||||
See documentation in docs/topics/email.rst
|
||||
"""
|
||||
|
||||
import logging
|
||||
from email import encoders as Encoders
|
||||
from email.mime.base import MIMEBase
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from scrapy import Spider
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.defer import process_chain, process_parallel
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
|
|
@ -64,7 +64,10 @@ class MiddlewareManager:
|
|||
for clspath in mwlist:
|
||||
try:
|
||||
mwcls = load_object(clspath)
|
||||
mw = create_instance(mwcls, settings, crawler)
|
||||
if crawler is not None:
|
||||
mw = build_from_crawler(mwcls, crawler)
|
||||
else:
|
||||
mw = build_from_settings(mwcls, settings)
|
||||
middlewares.append(mw)
|
||||
enabled.append(clspath)
|
||||
except NotConfigured as e:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Item pipeline
|
|||
|
||||
See documentation in docs/item-pipeline.rst
|
||||
"""
|
||||
|
||||
from typing import Any, List
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
|
|
|||
|
|
@ -3,12 +3,12 @@ Files Pipeline
|
|||
|
||||
See documentation in topics/media-pipeline.rst
|
||||
"""
|
||||
|
||||
import base64
|
||||
import functools
|
||||
import hashlib
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
|
|
@ -66,7 +66,7 @@ class FSFilesStore:
|
|||
absolute_path = self._get_filesystem_path(path)
|
||||
try:
|
||||
last_modified = absolute_path.stat().st_mtime
|
||||
except os.error:
|
||||
except OSError:
|
||||
return {}
|
||||
|
||||
with absolute_path.open("rb") as f:
|
||||
|
|
@ -340,7 +340,9 @@ class FilesPipeline(MediaPipeline):
|
|||
DEFAULT_FILES_URLS_FIELD = "file_urls"
|
||||
DEFAULT_FILES_RESULT_FIELD = "files"
|
||||
|
||||
def __init__(self, store_uri, download_func=None, settings=None):
|
||||
def __init__(
|
||||
self, store_uri: Union[str, PathLike], download_func=None, settings=None
|
||||
):
|
||||
store_uri = _to_string(store_uri)
|
||||
if not store_uri:
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ Images Pipeline
|
|||
|
||||
See documentation in topics/media-pipeline.rst
|
||||
"""
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from typing import Dict, Tuple
|
||||
from os import PathLike
|
||||
from typing import Dict, Tuple, Union
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
|
|
@ -53,7 +55,9 @@ class ImagesPipeline(FilesPipeline):
|
|||
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
|
||||
DEFAULT_IMAGES_RESULT_FIELD = "images"
|
||||
|
||||
def __init__(self, store_uri, download_func=None, settings=None):
|
||||
def __init__(
|
||||
self, store_uri: Union[str, PathLike], download_func=None, settings=None
|
||||
):
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
|
|
|
|||
|
|
@ -112,14 +112,14 @@ class MediaPipeline:
|
|||
info.downloading.add(fp)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
|
||||
dfd.addCallback(self._check_media_to_download, request, info, item=item)
|
||||
dfd.addErrback(self._log_exception)
|
||||
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
dfd.addErrback(
|
||||
lambda f: logger.error(
|
||||
f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
|
||||
)
|
||||
)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _log_exception(self, result):
|
||||
logger.exception(result)
|
||||
return result
|
||||
|
||||
def _modify_media_request(self, request):
|
||||
if self.handle_httpstatus_list:
|
||||
request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import hashlib
|
||||
import logging
|
||||
|
||||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -72,9 +72,8 @@ class ScrapyPriorityQueue:
|
|||
self.curprio = min(startprios)
|
||||
|
||||
def qfactory(self, key):
|
||||
return create_instance(
|
||||
return build_from_crawler(
|
||||
self.downstream_queue_cls,
|
||||
None,
|
||||
self.crawler,
|
||||
self.key + "/" + str(key),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
This module implements a class which returns the appropriate Response class
|
||||
based on different criteria.
|
||||
"""
|
||||
|
||||
from io import StringIO
|
||||
from mimetypes import MimeTypes
|
||||
from pkgutil import get_data
|
||||
|
|
@ -21,9 +22,9 @@ class ResponseTypes:
|
|||
"application/xhtml+xml": "scrapy.http.HtmlResponse",
|
||||
"application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse",
|
||||
"application/xml": "scrapy.http.XmlResponse",
|
||||
"application/json": "scrapy.http.TextResponse",
|
||||
"application/x-json": "scrapy.http.TextResponse",
|
||||
"application/json-amazonui-streaming": "scrapy.http.TextResponse",
|
||||
"application/json": "scrapy.http.JsonResponse",
|
||||
"application/x-json": "scrapy.http.JsonResponse",
|
||||
"application/json-amazonui-streaming": "scrapy.http.JsonResponse",
|
||||
"application/javascript": "scrapy.http.TextResponse",
|
||||
"application/x-javascript": "scrapy.http.TextResponse",
|
||||
"text/xml": "scrapy.http.XmlResponse",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
XPath selectors based on lxml
|
||||
"""
|
||||
|
||||
from typing import Any, Optional, Type, Union
|
||||
|
||||
from parsel import Selector as _ParselSelector
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int:
|
|||
|
||||
|
||||
class SettingsAttribute:
|
||||
|
||||
"""Class for storing data related to settings attributes.
|
||||
|
||||
This class is intended for internal usage, you should try Settings class
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ REFERER_ENABLED = True
|
|||
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
See documentation in docs/topics/shell.rst
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import signal
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ HttpError Spider Middleware
|
|||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Offsite Spider Middleware
|
|||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
RefererMiddleware: populates Request referer field, based on the Response which
|
||||
originated it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Base class for Scrapy spiders
|
|||
|
||||
See documentation in docs/topics/spiders.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class CrawlSpider(Spider):
|
|||
url=link.url,
|
||||
callback=self._callback,
|
||||
errback=self._errback,
|
||||
meta=dict(rule=rule_index, link_text=link.text),
|
||||
meta={"rule": rule_index, "link_text": link.text},
|
||||
)
|
||||
|
||||
def _requests_to_follow(self, response):
|
||||
|
|
@ -131,8 +131,7 @@ class CrawlSpider(Spider):
|
|||
def _handle_failure(self, failure, errback):
|
||||
if errback:
|
||||
results = errback(failure) or ()
|
||||
for request_or_item in iterate_spider_output(results):
|
||||
yield request_or_item
|
||||
yield from iterate_spider_output(results)
|
||||
|
||||
def _compile_rules(self):
|
||||
self._rules = []
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ for scraping from an XML feed.
|
|||
|
||||
See documentation in docs/topics/spiders.rst
|
||||
"""
|
||||
|
||||
from scrapy.exceptions import NotConfigured, NotSupported
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.iterators import csviter, xmliter
|
||||
from scrapy.utils.iterators import csviter, xmliter_lxml
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
|
|
@ -58,8 +59,7 @@ class XMLFeedSpider(Spider):
|
|||
|
||||
for selector in nodes:
|
||||
ret = iterate_spider_output(self.parse_node(response, selector))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
yield from self.process_results(response, ret)
|
||||
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, "parse_node"):
|
||||
|
|
@ -84,7 +84,7 @@ class XMLFeedSpider(Spider):
|
|||
return self.parse_nodes(response, nodes)
|
||||
|
||||
def _iternodes(self, response):
|
||||
for node in xmliter(response, self.itertag):
|
||||
for node in xmliter_lxml(response, self.itertag):
|
||||
self._register_namespaces(node)
|
||||
yield node
|
||||
|
||||
|
|
@ -133,8 +133,7 @@ class CSVFeedSpider(Spider):
|
|||
response, self.delimiter, self.headers, quotechar=self.quotechar
|
||||
):
|
||||
ret = iterate_spider_output(self.parse_row(response, row))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
yield from self.process_results(response, ret)
|
||||
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, "parse_row"):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
import logging
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from scrapy.http import Request, XmlResponse
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
|
||||
from scrapy.utils.gz import gunzip, gzip_magic_number
|
||||
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -14,6 +22,19 @@ class SitemapSpider(Spider):
|
|||
sitemap_rules = [("", "parse")]
|
||||
sitemap_follow = [""]
|
||||
sitemap_alternate_links = False
|
||||
_max_size: int
|
||||
_warn_size: int
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self":
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider._max_size = getattr(
|
||||
spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE")
|
||||
)
|
||||
spider._warn_size = getattr(
|
||||
spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE")
|
||||
)
|
||||
return spider
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
|
|
@ -33,8 +54,7 @@ class SitemapSpider(Spider):
|
|||
attributes, for example, you can filter locs with lastmod greater
|
||||
than a given date (see docs).
|
||||
"""
|
||||
for entry in entries:
|
||||
yield entry
|
||||
yield from entries
|
||||
|
||||
def _parse_sitemap(self, response):
|
||||
if response.url.endswith("/robots.txt"):
|
||||
|
|
@ -71,7 +91,19 @@ class SitemapSpider(Spider):
|
|||
if isinstance(response, XmlResponse):
|
||||
return response.body
|
||||
if gzip_magic_number(response):
|
||||
return gunzip(response.body)
|
||||
uncompressed_size = len(response.body)
|
||||
max_size = response.meta.get("download_maxsize", self._max_size)
|
||||
warn_size = response.meta.get("download_warnsize", self._warn_size)
|
||||
try:
|
||||
body = gunzip(response.body, max_size=max_size)
|
||||
except _DecompressionMaxSizeExceeded:
|
||||
return None
|
||||
if uncompressed_size < warn_size <= len(body):
|
||||
logger.warning(
|
||||
f"{response} body size after decompression ({len(body)} B) "
|
||||
f"is larger than the download warning size ({warn_size} B)."
|
||||
)
|
||||
return body
|
||||
# actual gzipped sitemap files are decompressed above ;
|
||||
# if we are here (response body is not gzipped)
|
||||
# and have a response for .xml.gz,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Scrapy extension for collecting scraping stats
|
||||
"""
|
||||
|
||||
import logging
|
||||
import pprint
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
|
|
|||
|
|
@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True
|
|||
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
|
||||
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
FEED_EXPORT_ENCODING = "utf-8"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
import zlib
|
||||
from io import BytesIO
|
||||
|
||||
try:
|
||||
import brotli
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import zstandard
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
_CHUNK_SIZE = 65536 # 64 KiB
|
||||
|
||||
|
||||
class _DecompressionMaxSizeExceeded(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
|
||||
decompressor = zlib.decompressobj()
|
||||
raw_decompressor = zlib.decompressobj(wbits=-15)
|
||||
input_stream = BytesIO(data)
|
||||
output_stream = BytesIO()
|
||||
output_chunk = b"."
|
||||
decompressed_size = 0
|
||||
while output_chunk:
|
||||
input_chunk = input_stream.read(_CHUNK_SIZE)
|
||||
try:
|
||||
output_chunk = decompressor.decompress(input_chunk)
|
||||
except zlib.error:
|
||||
if decompressor != raw_decompressor:
|
||||
# ugly hack to work with raw deflate content that may
|
||||
# be sent by microsoft servers. For more information, see:
|
||||
# http://carsten.codimi.de/gzip.yaws/
|
||||
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
|
||||
# http://www.gzip.org/zlib/zlib_faq.html#faq38
|
||||
decompressor = raw_decompressor
|
||||
output_chunk = decompressor.decompress(input_chunk)
|
||||
else:
|
||||
raise
|
||||
decompressed_size += len(output_chunk)
|
||||
if max_size and decompressed_size > max_size:
|
||||
raise _DecompressionMaxSizeExceeded(
|
||||
f"The number of bytes decompressed so far "
|
||||
f"({decompressed_size} B) exceed the specified maximum "
|
||||
f"({max_size} B)."
|
||||
)
|
||||
output_stream.write(output_chunk)
|
||||
output_stream.seek(0)
|
||||
return output_stream.read()
|
||||
|
||||
|
||||
def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
|
||||
decompressor = brotli.Decompressor()
|
||||
input_stream = BytesIO(data)
|
||||
output_stream = BytesIO()
|
||||
output_chunk = b"."
|
||||
decompressed_size = 0
|
||||
while output_chunk:
|
||||
input_chunk = input_stream.read(_CHUNK_SIZE)
|
||||
output_chunk = decompressor.process(input_chunk)
|
||||
decompressed_size += len(output_chunk)
|
||||
if max_size and decompressed_size > max_size:
|
||||
raise _DecompressionMaxSizeExceeded(
|
||||
f"The number of bytes decompressed so far "
|
||||
f"({decompressed_size} B) exceed the specified maximum "
|
||||
f"({max_size} B)."
|
||||
)
|
||||
output_stream.write(output_chunk)
|
||||
output_stream.seek(0)
|
||||
return output_stream.read()
|
||||
|
||||
|
||||
def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
|
||||
decompressor = zstandard.ZstdDecompressor()
|
||||
stream_reader = decompressor.stream_reader(BytesIO(data))
|
||||
output_stream = BytesIO()
|
||||
output_chunk = b"."
|
||||
decompressed_size = 0
|
||||
while output_chunk:
|
||||
output_chunk = stream_reader.read(_CHUNK_SIZE)
|
||||
decompressed_size += len(output_chunk)
|
||||
if max_size and decompressed_size > max_size:
|
||||
raise _DecompressionMaxSizeExceeded(
|
||||
f"The number of bytes decompressed so far "
|
||||
f"({decompressed_size} B) exceed the specified maximum "
|
||||
f"({max_size} B)."
|
||||
)
|
||||
output_stream.write(output_chunk)
|
||||
output_stream.seek(0)
|
||||
return output_stream.read()
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Helper functions for dealing with Twisted deferreds
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
from asyncio import Future
|
||||
|
|
@ -304,13 +305,11 @@ _T = TypeVar("_T")
|
|||
|
||||
|
||||
@overload
|
||||
def deferred_from_coro(o: _CT) -> Deferred:
|
||||
...
|
||||
def deferred_from_coro(o: _CT) -> Deferred: ...
|
||||
|
||||
|
||||
@overload
|
||||
def deferred_from_coro(o: _T) -> _T:
|
||||
...
|
||||
def deferred_from_coro(o: _T) -> _T: ...
|
||||
|
||||
|
||||
def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
|
||||
|
|
|
|||
|
|
@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = []
|
|||
|
||||
|
||||
@overload
|
||||
def update_classpath(path: str) -> str:
|
||||
...
|
||||
def update_classpath(path: str) -> str: ...
|
||||
|
||||
|
||||
@overload
|
||||
def update_classpath(path: Any) -> Any:
|
||||
...
|
||||
def update_classpath(path: Any) -> Any: ...
|
||||
|
||||
|
||||
def update_classpath(path: Any) -> Any:
|
||||
|
|
|
|||
|
|
@ -1,31 +1,41 @@
|
|||
import struct
|
||||
from gzip import GzipFile
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
from scrapy.http import Response
|
||||
|
||||
from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
|
||||
|
||||
def gunzip(data: bytes) -> bytes:
|
||||
|
||||
def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
|
||||
"""Gunzip the given data and return as much data as possible.
|
||||
|
||||
This is resilient to CRC checksum errors.
|
||||
"""
|
||||
f = GzipFile(fileobj=BytesIO(data))
|
||||
output_list: List[bytes] = []
|
||||
output_stream = BytesIO()
|
||||
chunk = b"."
|
||||
decompressed_size = 0
|
||||
while chunk:
|
||||
try:
|
||||
chunk = f.read1(8196)
|
||||
output_list.append(chunk)
|
||||
chunk = f.read1(_CHUNK_SIZE)
|
||||
except (OSError, EOFError, struct.error):
|
||||
# complete only if there is some data, otherwise re-raise
|
||||
# see issue 87 about catching struct.error
|
||||
# some pages are quite small so output_list is empty
|
||||
if output_list:
|
||||
# some pages are quite small so output_stream is empty
|
||||
if output_stream.getbuffer().nbytes > 0:
|
||||
break
|
||||
raise
|
||||
return b"".join(output_list)
|
||||
decompressed_size += len(chunk)
|
||||
if max_size and decompressed_size > max_size:
|
||||
raise _DecompressionMaxSizeExceeded(
|
||||
f"The number of bytes decompressed so far "
|
||||
f"({decompressed_size} B) exceed the specified maximum "
|
||||
f"({max_size} B)."
|
||||
)
|
||||
output_stream.write(chunk)
|
||||
output_stream.seek(0)
|
||||
return output_stream.read()
|
||||
|
||||
|
||||
def gzip_magic_number(response: Response) -> bool:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,11 @@ from typing import (
|
|||
cast,
|
||||
overload,
|
||||
)
|
||||
from warnings import warn
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.python import re_rsearch, to_unicode
|
||||
|
|
@ -38,6 +42,16 @@ def xmliter(
|
|||
- a unicode string
|
||||
- a string encoded as utf-8
|
||||
"""
|
||||
warn(
|
||||
(
|
||||
"xmliter is deprecated and its use strongly discouraged because "
|
||||
"it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See "
|
||||
"https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9"
|
||||
),
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
nodename_patt = re.escape(nodename)
|
||||
|
||||
DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S)
|
||||
|
|
@ -81,15 +95,34 @@ def xmliter_lxml(
|
|||
namespace: Optional[str] = None,
|
||||
prefix: str = "x",
|
||||
) -> Generator[Selector, Any, None]:
|
||||
from lxml import etree
|
||||
|
||||
reader = _StreamReader(obj)
|
||||
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
|
||||
iterable = etree.iterparse(
|
||||
cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding
|
||||
cast("SupportsReadClose[bytes]", reader),
|
||||
encoding=reader.encoding,
|
||||
events=("end", "start-ns"),
|
||||
huge_tree=True,
|
||||
)
|
||||
selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
|
||||
for _, node in iterable:
|
||||
needs_namespace_resolution = not namespace and ":" in nodename
|
||||
if needs_namespace_resolution:
|
||||
prefix, nodename = nodename.split(":", maxsplit=1)
|
||||
for event, data in iterable:
|
||||
if event == "start-ns":
|
||||
assert isinstance(data, tuple)
|
||||
if needs_namespace_resolution:
|
||||
_prefix, _namespace = data
|
||||
if _prefix != prefix:
|
||||
continue
|
||||
namespace = _namespace
|
||||
needs_namespace_resolution = False
|
||||
selxpath = f"//{prefix}:{nodename}"
|
||||
tag = f"{{{namespace}}}{nodename}"
|
||||
continue
|
||||
assert isinstance(data, etree._Element)
|
||||
node = data
|
||||
if node.tag != tag:
|
||||
continue
|
||||
nodetext = etree.tostring(node, encoding="unicode")
|
||||
node.clear()
|
||||
xs = Selector(text=nodetext, type="xml")
|
||||
|
|
@ -192,18 +225,17 @@ def csviter(
|
|||
|
||||
|
||||
@overload
|
||||
def _body_or_str(obj: Union[Response, str, bytes]) -> str:
|
||||
...
|
||||
def _body_or_str(obj: Union[Response, str, bytes]) -> str: ...
|
||||
|
||||
|
||||
@overload
|
||||
def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
|
||||
...
|
||||
def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ...
|
||||
|
||||
|
||||
@overload
|
||||
def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
|
||||
...
|
||||
def _body_or_str(
|
||||
obj: Union[Response, str, bytes], unicode: Literal[False]
|
||||
) -> bytes: ...
|
||||
|
||||
|
||||
def _body_or_str(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Helper functions which don't fit anywhere else"""
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import inspect
|
||||
|
|
@ -25,6 +26,7 @@ from typing import (
|
|||
cast,
|
||||
)
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
|
||||
|
|
@ -142,6 +144,13 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
"""
|
||||
warnings.warn(
|
||||
"The create_instance() function is deprecated. "
|
||||
"Please use build_from_crawler() or build_from_settings() instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
raise ValueError("Specify at least one of settings and crawler.")
|
||||
|
|
@ -160,6 +169,45 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
return instance
|
||||
|
||||
|
||||
def build_from_crawler(objcls, crawler, /, *args, **kwargs):
|
||||
"""Construct a class instance using its ``from_crawler`` constructor.
|
||||
|
||||
``*args`` and ``**kwargs`` are forwarded to the constructor.
|
||||
|
||||
Raises ``TypeError`` if the resulting instance is ``None``.
|
||||
"""
|
||||
if hasattr(objcls, "from_crawler"):
|
||||
instance = objcls.from_crawler(crawler, *args, **kwargs)
|
||||
method_name = "from_crawler"
|
||||
elif hasattr(objcls, "from_settings"):
|
||||
instance = objcls.from_settings(crawler.settings, *args, **kwargs)
|
||||
method_name = "from_settings"
|
||||
else:
|
||||
instance = objcls(*args, **kwargs)
|
||||
method_name = "__new__"
|
||||
if instance is None:
|
||||
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
|
||||
def build_from_settings(objcls, settings, /, *args, **kwargs):
|
||||
"""Construct a class instance using its ``from_settings`` constructor.
|
||||
|
||||
``*args`` and ``**kwargs`` are forwarded to the constructor.
|
||||
|
||||
Raises ``TypeError`` if the resulting instance is ``None``.
|
||||
"""
|
||||
if hasattr(objcls, "from_settings"):
|
||||
instance = objcls.from_settings(settings, *args, **kwargs)
|
||||
method_name = "from_settings"
|
||||
else:
|
||||
instance = objcls(*args, **kwargs)
|
||||
method_name = "__new__"
|
||||
if instance is None:
|
||||
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
|
||||
@contextmanager
|
||||
def set_environ(**kwargs: str) -> Generator[None, Any, None]:
|
||||
"""Temporarily set environment variables inside the context manager and
|
||||
|
|
|
|||
|
|
@ -24,7 +24,11 @@ def install_shutdown_handlers(
|
|||
(e.g. Pdb)
|
||||
"""
|
||||
signal.signal(signal.SIGTERM, function)
|
||||
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
|
||||
if (
|
||||
signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable
|
||||
== signal.default_int_handler
|
||||
or override_sigint
|
||||
):
|
||||
signal.signal(signal.SIGINT, function)
|
||||
# Catch Ctrl-Break in windows
|
||||
if hasattr(signal, "SIGBREAK"):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
This module contains essential stuff that should've come with Python itself ;)
|
||||
"""
|
||||
|
||||
import collections.abc
|
||||
import gc
|
||||
import inspect
|
||||
|
|
@ -57,8 +58,7 @@ def iflatten(x: Iterable) -> Iterable:
|
|||
Similar to ``.flatten()``, but returns iterator instead"""
|
||||
for el in x:
|
||||
if is_listlike(el):
|
||||
for el_ in iflatten(el):
|
||||
yield el_
|
||||
yield from iflatten(el)
|
||||
else:
|
||||
yield el
|
||||
|
||||
|
|
@ -162,7 +162,7 @@ def re_rsearch(
|
|||
pattern = re.compile(pattern)
|
||||
|
||||
for chunk, offset in _chunk_iter():
|
||||
matches = [match for match in pattern.finditer(chunk)]
|
||||
matches = list(pattern.finditer(chunk))
|
||||
if matches:
|
||||
start, end = matches[-1].span()
|
||||
return offset + start, offset + end
|
||||
|
|
@ -286,13 +286,11 @@ def equal_attributes(
|
|||
|
||||
|
||||
@overload
|
||||
def without_none_values(iterable: Mapping) -> dict:
|
||||
...
|
||||
def without_none_values(iterable: Mapping) -> dict: ...
|
||||
|
||||
|
||||
@overload
|
||||
def without_none_values(iterable: Iterable) -> Iterable:
|
||||
...
|
||||
def without_none_values(iterable: Iterable) -> Iterable: ...
|
||||
|
||||
|
||||
def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,6 @@ from scrapy.utils.python import to_bytes, to_unicode
|
|||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
|
||||
_deprecated_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _serialize_headers(
|
||||
headers: Iterable[bytes], request: Request
|
||||
|
|
@ -44,125 +41,12 @@ def _serialize_headers(
|
|||
for header in headers:
|
||||
if header in request.headers:
|
||||
yield header
|
||||
for value in request.headers.getlist(header):
|
||||
yield value
|
||||
yield from request.headers.getlist(header)
|
||||
|
||||
|
||||
def request_fingerprint(
|
||||
request: Request,
|
||||
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
|
||||
keep_fragments: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Return the request fingerprint as an hexadecimal string.
|
||||
|
||||
The request fingerprint is a hash that uniquely identifies the resource the
|
||||
request points to. For example, take the following two urls:
|
||||
|
||||
http://www.example.com/query?id=111&cat=222
|
||||
http://www.example.com/query?cat=222&id=111
|
||||
|
||||
Even though those are two different URLs both point to the same resource
|
||||
and are equivalent (i.e. they should return the same response).
|
||||
|
||||
Another example are cookies used to store session ids. Suppose the
|
||||
following page is only accessible to authenticated users:
|
||||
|
||||
http://www.example.com/members/offers.html
|
||||
|
||||
Lots of sites use a cookie to store the session id, which adds a random
|
||||
component to the HTTP Request and thus should be ignored when calculating
|
||||
the fingerprint.
|
||||
|
||||
For this reason, request headers are ignored by default when calculating
|
||||
the fingerprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
"""
|
||||
if include_headers or keep_fragments:
|
||||
message = (
|
||||
"Call to deprecated function "
|
||||
"scrapy.utils.request.request_fingerprint().\n"
|
||||
"\n"
|
||||
"If you are using this function in a Scrapy component because you "
|
||||
"need a non-default fingerprinting algorithm, and you are OK "
|
||||
"with that non-default fingerprinting algorithm being used by "
|
||||
"all Scrapy components and not just the one calling this "
|
||||
"function, use crawler.request_fingerprinter.fingerprint() "
|
||||
"instead in your Scrapy component (you can get the crawler "
|
||||
"object from the 'from_crawler' class method), and use the "
|
||||
"'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
|
||||
"non-default fingerprinting algorithm.\n"
|
||||
"\n"
|
||||
"Otherwise, consider using the "
|
||||
"scrapy.utils.request.fingerprint() function instead.\n"
|
||||
"\n"
|
||||
"If you switch to 'fingerprint()', or assign the "
|
||||
"'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
|
||||
"'fingerprint()', the generated fingerprints will not only be "
|
||||
"bytes instead of a string, but they will also be different from "
|
||||
"those generated by 'request_fingerprint()'. Before you switch, "
|
||||
"make sure that you understand the consequences of this (e.g. "
|
||||
"cache invalidation) and are OK with them; otherwise, consider "
|
||||
"implementing your own function which returns the same "
|
||||
"fingerprints as the deprecated 'request_fingerprint()' function."
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"Call to deprecated function "
|
||||
"scrapy.utils.request.request_fingerprint().\n"
|
||||
"\n"
|
||||
"If you are using this function in a Scrapy component, and you "
|
||||
"are OK with users of your component changing the fingerprinting "
|
||||
"algorithm through settings, use "
|
||||
"crawler.request_fingerprinter.fingerprint() instead in your "
|
||||
"Scrapy component (you can get the crawler object from the "
|
||||
"'from_crawler' class method).\n"
|
||||
"\n"
|
||||
"Otherwise, consider using the "
|
||||
"scrapy.utils.request.fingerprint() function instead.\n"
|
||||
"\n"
|
||||
"Either way, the resulting fingerprints will be returned as "
|
||||
"bytes, not as a string, and they will also be different from "
|
||||
"those generated by 'request_fingerprint()'. Before you switch, "
|
||||
"make sure that you understand the consequences of this (e.g. "
|
||||
"cache invalidation) and are OK with them; otherwise, consider "
|
||||
"implementing your own function which returns the same "
|
||||
"fingerprints as the deprecated 'request_fingerprint()' function."
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
processed_include_headers: Optional[Tuple[bytes, ...]] = None
|
||||
if include_headers:
|
||||
processed_include_headers = tuple(
|
||||
to_bytes(h.lower()) for h in sorted(include_headers)
|
||||
)
|
||||
cache = _deprecated_fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (processed_include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(
|
||||
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
|
||||
)
|
||||
fp.update(request.body or b"")
|
||||
if processed_include_headers:
|
||||
for part in _serialize_headers(processed_include_headers, request):
|
||||
fp.update(part)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
return bytes.fromhex(request_fingerprint(*args, **kwargs))
|
||||
|
||||
|
||||
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
|
||||
_fingerprint_cache: (
|
||||
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
|
||||
)
|
||||
_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
|
|
@ -232,8 +116,7 @@ def fingerprint(
|
|||
|
||||
|
||||
class RequestFingerprinterProtocol(Protocol):
|
||||
def fingerprint(self, request: Request) -> bytes:
|
||||
...
|
||||
def fingerprint(self, request: Request) -> bytes: ...
|
||||
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
|
@ -259,33 +142,15 @@ class RequestFingerprinter:
|
|||
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
|
||||
)
|
||||
else:
|
||||
implementation = "2.6"
|
||||
if implementation == "2.6":
|
||||
implementation = "SENTINEL"
|
||||
|
||||
if implementation != "SENTINEL":
|
||||
message = (
|
||||
"'2.6' is a deprecated value for the "
|
||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
|
||||
"\n"
|
||||
"It is also the default value. In other words, it is normal "
|
||||
"to get this warning if you have not defined a value for the "
|
||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
|
||||
"for backward compatibility reasons, but it will change in a "
|
||||
"future version of Scrapy.\n"
|
||||
"\n"
|
||||
"See the documentation of the "
|
||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
|
||||
"information on how to handle this deprecation."
|
||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
|
||||
"And it will be removed in future version of Scrapy."
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._fingerprint = _request_fingerprint_as_bytes
|
||||
elif implementation == "2.7":
|
||||
self._fingerprint = fingerprint
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Got an invalid value on setting "
|
||||
f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
|
||||
f"{implementation!r}. Valid values are '2.6' (deprecated) "
|
||||
f"and '2.7'."
|
||||
)
|
||||
self._fingerprint = fingerprint
|
||||
|
||||
def fingerprint(self, request: Request) -> bytes:
|
||||
return self._fingerprint(request)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
This module provides some useful functions for working with
|
||||
scrapy.http.Response objects
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
|
|
@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
|
|||
return _baseurl_cache[response]
|
||||
|
||||
|
||||
_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
|
||||
WeakKeyDictionary()
|
||||
)
|
||||
_metaref_cache: (
|
||||
"WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]"
|
||||
) = WeakKeyDictionary()
|
||||
|
||||
|
||||
def get_meta_refresh(
|
||||
|
|
@ -54,6 +55,18 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
|
|||
return f"{status_int} {to_unicode(message)}"
|
||||
|
||||
|
||||
def _remove_html_comments(body):
|
||||
start = body.find(b"<!--")
|
||||
while start != -1:
|
||||
end = body.find(b"-->", start + 1)
|
||||
if end == -1:
|
||||
return body[:start]
|
||||
else:
|
||||
body = body[:start] + body[end + 3 :]
|
||||
start = body.find(b"<!--")
|
||||
return body
|
||||
|
||||
|
||||
def open_in_browser(
|
||||
response: Union[
|
||||
"scrapy.http.response.html.HtmlResponse",
|
||||
|
|
@ -61,8 +74,21 @@ def open_in_browser(
|
|||
],
|
||||
_openfunc: Callable[[str], Any] = webbrowser.open,
|
||||
) -> Any:
|
||||
"""Open the given response in a local web browser, populating the <base>
|
||||
tag for external links to work
|
||||
"""Open *response* in a local web browser, adjusting the `base tag`_ for
|
||||
external links to work, e.g. so that images and styles are displayed.
|
||||
|
||||
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
||||
def parse_details(self, response):
|
||||
if "item name" not in response.body:
|
||||
open_in_browser(response)
|
||||
"""
|
||||
from scrapy.http import HtmlResponse, TextResponse
|
||||
|
||||
|
|
@ -70,9 +96,9 @@ def open_in_browser(
|
|||
body = response.body
|
||||
if isinstance(response, HtmlResponse):
|
||||
if b"<base" not in body:
|
||||
repl = rf'\1<base href="{response.url}">'
|
||||
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
|
||||
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
|
||||
_remove_html_comments(body)
|
||||
repl = rf'\0<base href="{response.url}">'
|
||||
body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)
|
||||
ext = ".html"
|
||||
elif isinstance(response, TextResponse):
|
||||
ext = ".txt"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"""Helper functions for working with signals"""
|
||||
|
||||
import collections.abc
|
||||
import logging
|
||||
from typing import Any as TypingAny
|
||||
|
|
@ -97,7 +98,10 @@ def send_catch_log_deferred(
|
|||
robustApply, receiver, signal=signal, sender=sender, *arguments, **named
|
||||
)
|
||||
d.addErrback(logerror, receiver)
|
||||
d.addBoth(lambda result: (receiver, result))
|
||||
# TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
|
||||
d.addBoth(
|
||||
lambda result: (receiver, result) # pylint: disable=cell-var-from-loop
|
||||
)
|
||||
dfds.append(d)
|
||||
d = DeferredList(dfds)
|
||||
d.addCallback(lambda out: [x[1] for x in out])
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Module for processing Sitemaps.
|
|||
Note: The main purpose of this module is to provide support for the
|
||||
SitemapSpider, its API is subject to change without notice.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Generator, Iterator, Optional
|
||||
from urllib.parse import urljoin
|
||||
|
||||
|
|
|
|||
|
|
@ -34,18 +34,15 @@ _T = TypeVar("_T")
|
|||
|
||||
# https://stackoverflow.com/questions/60222982
|
||||
@overload
|
||||
def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc]
|
||||
...
|
||||
def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap]
|
||||
|
||||
|
||||
@overload
|
||||
def iterate_spider_output(result: CoroutineType) -> Deferred:
|
||||
...
|
||||
def iterate_spider_output(result: CoroutineType) -> Deferred: ...
|
||||
|
||||
|
||||
@overload
|
||||
def iterate_spider_output(result: _T) -> Iterable:
|
||||
...
|
||||
def iterate_spider_output(result: _T) -> Iterable: ...
|
||||
|
||||
|
||||
def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
|
||||
|
|
@ -83,8 +80,7 @@ def spidercls_for_request(
|
|||
default_spidercls: Type[Spider],
|
||||
log_none: bool = ...,
|
||||
log_multiple: bool = ...,
|
||||
) -> Type[Spider]:
|
||||
...
|
||||
) -> Type[Spider]: ...
|
||||
|
||||
|
||||
@overload
|
||||
|
|
@ -94,8 +90,7 @@ def spidercls_for_request(
|
|||
default_spidercls: Literal[None],
|
||||
log_none: bool = ...,
|
||||
log_multiple: bool = ...,
|
||||
) -> Optional[Type[Spider]]:
|
||||
...
|
||||
) -> Optional[Type[Spider]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
|
|
@ -105,8 +100,7 @@ def spidercls_for_request(
|
|||
*,
|
||||
log_none: bool = ...,
|
||||
log_multiple: bool = ...,
|
||||
) -> Optional[Type[Spider]]:
|
||||
...
|
||||
) -> Optional[Type[Spider]]: ...
|
||||
|
||||
|
||||
def spidercls_for_request(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Any, Optional
|
||||
|
||||
import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped]
|
||||
import OpenSSL._util as pyOpenSSLutil
|
||||
import OpenSSL.SSL
|
||||
import OpenSSL.version
|
||||
from OpenSSL.crypto import X509Name
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue