...
Kenmore White 17" Microwave
...
@@ -821,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
- ... print(" properties: %s" % (props.getall()))
+ ... print(f" properties: {props.getall()}")
... print("")
current scope: ['http://schema.org/Product']
@@ -989,7 +991,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel.xpath("//h1")
2. Extract the text of all ``
`` elements from an HTML response body,
- returning a list of unicode strings::
+ returning a list of strings::
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5178f272f..0b290598f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -98,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
+
+Import paths and classes
+========================
+
+.. versionadded:: 2.4.0
+
+When a setting references a callable object to be imported by Scrapy, such as a
+class or a function, there are two different ways you can specify that object:
+
+- As a string containing the import path of that object
+
+- As the object itself
+
+For example::
+
+ from mybot.pipelines.validate import ValidateMyItem
+ ITEM_PIPELINES = {
+ # passing the classname...
+ ValidateMyItem: 300,
+ # ...equals passing the class path
+ 'mybot.pipelines.validate.ValidateMyItem': 300,
+ }
+
+.. note:: Passing non-callable objects is not supported.
+
+
How to access settings
======================
@@ -110,7 +136,7 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com']
def parse(self, response):
- print("Existing settings: %s" % self.settings.attributes.keys())
+ print(f"Existing settings: {self.settings.attributes.keys()}")
.. note::
The ``settings`` attribute is set in the base Spider class after the spider
@@ -216,6 +242,32 @@ Default: ``None``
The name of the region associated with the AWS client.
+.. setting:: ASYNCIO_EVENT_LOOP
+
+ASYNCIO_EVENT_LOOP
+------------------
+
+Default: ``None``
+
+Import path of a given ``asyncio`` event loop class.
+
+If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
+asyncio event loop to be used with it. Set the setting to the import path of the
+desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
+event loop will be used.
+
+If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
+function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
+class to be used.
+
+Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
+
+.. caution:: Please be aware that, when using a non-default event loop
+ (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
+ :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
+ :func:`asyncio.set_event_loop`, which will set the specified event loop
+ as the current loop for the current OS thread.
+
.. setting:: BOT_NAME
BOT_NAME
@@ -306,6 +358,11 @@ Default::
The default headers used for Scrapy HTTP Requests. They're populated in the
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
+.. caution:: Cookies set via the ``Cookie`` header are not considered by the
+ :ref:`cookies-mw`. If you need to set cookies for a request, use the
+ :class:`Request.cookies ` parameter. This is a known
+ current limitation that is being worked on.
+
.. setting:: DEPTH_LIMIT
DEPTH_LIMIT
@@ -469,7 +526,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
-.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@@ -600,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE
Default::
{
+ 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@@ -620,6 +678,45 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
+.. _http2:
+
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
+
+#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
+ enable HTTP/2 support in Twisted.
+
+#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
+
+ DOWNLOAD_HANDLERS = {
+ 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+ }
+
+.. warning::
+
+ HTTP/2 support in Scrapy is experimental, and not yet recommended for
+ production environments. Future Scrapy versions may introduce related
+ changes without a deprecation period or warning.
+
+.. note::
+
+ Known limitations of the current HTTP/2 implementation of Scrapy include:
+
+ - No support for HTTP/2 Cleartext (h2c), since no major browser supports
+ HTTP/2 unencrypted (refer `http2 faq`_).
+
+ - No setting to specify a maximum `frame size`_ larger than the default
+ value, 16384. Connections to servers that send a larger frame will
+ fail.
+
+ - No support for `server pushes`_, which are ignored.
+
+ - No support for the :signal:`bytes_received` and
+ :signal:`headers_received` signals.
+
+.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
+.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
+.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
+
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@@ -697,6 +794,15 @@ Optionally, this can be set per-request basis by using the
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
+.. warning::
+
+ This setting is ignored by the
+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
+ download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
+ error, the corresponding HTTP/2 connection may be corrupted, affecting other
+ requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
+ failure is always raised for every request that was using that connection.
+
.. setting:: DUPEFILTER_CLASS
DUPEFILTER_CLASS
@@ -786,6 +892,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
temporary files before uploading with :ref:`FTP feed storage ` and
:ref:`Amazon S3 `.
+.. setting:: FEED_STORAGE_GCS_ACL
+
+FEED_STORAGE_GCS_ACL
+--------------------
+
+The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `.
+For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_.
+
.. setting:: FTP_PASSIVE_MODE
FTP_PASSIVE_MODE
@@ -825,6 +939,15 @@ Default: ``"anonymous"``
The username to use for FTP connections when there is no ``"ftp_user"``
in ``Request`` meta.
+.. setting:: GCS_PROJECT_ID
+
+GCS_PROJECT_ID
+-----------------
+
+Default: ``None``
+
+The Project ID that will be used when storing data on `Google Cloud Storage`_.
+
.. setting:: ITEM_PIPELINES
ITEM_PIPELINES
@@ -1013,8 +1136,6 @@ See :ref:`topics-extensions-ref-memusage`.
MEMUSAGE_CHECK_INTERVAL_SECONDS
-------------------------------
-.. versionadded:: 1.1
-
Default: ``60.0``
Scope: ``scrapy.extensions.memusage``
@@ -1116,20 +1237,6 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
-.. setting:: RETRY_PRIORITY_ADJUST
-
-RETRY_PRIORITY_ADJUST
----------------------
-
-Default: ``-1``
-
-Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
-
-Adjust retry request priority relative to original request:
-
-- a positive priority adjust means higher priority.
-- **a negative priority adjust (default) means lower priority.**
-
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY
@@ -1177,7 +1284,8 @@ SCHEDULER
Default: ``'scrapy.core.scheduler.Scheduler'``
-The scheduler to use for crawling.
+The scheduler class to be used for crawling.
+See the :ref:`topics-scheduler` topic for details.
.. setting:: SCHEDULER_DEBUG
@@ -1295,8 +1403,6 @@ The class that will be used for loading spiders, which must implement the
SPIDER_LOADER_WARN_ONLY
-----------------------
-.. versionadded:: 1.3.3
-
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
@@ -1517,7 +1623,7 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
-the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
+the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@@ -1540,7 +1646,7 @@ case to see how to enable and use them.
.. settingslist::
-
.. _Amazon web services: https://aws.amazon.com/
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
+.. _Google Cloud Storage: https://cloud.google.com/storage/
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 0f46f1c87..b910fc453 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -95,20 +95,21 @@ convenience.
Available Shortcuts
-------------------
- * ``shelp()`` - print a help with the list of available objects and shortcuts
+- ``shelp()`` - print a help with the list of available objects and
+ shortcuts
- * ``fetch(url[, redirect=True])`` - fetch a new response from the given
- URL and update all related objects accordingly. You can optionaly ask for
- HTTP 3xx redirections to not be followed by passing ``redirect=False``
+- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
+ and update all related objects accordingly. You can optionaly ask for HTTP
+ 3xx redirections to not be followed by passing ``redirect=False``
- * ``fetch(request)`` - fetch a new response from the given request and
- update all related objects accordingly.
+- ``fetch(request)`` - fetch a new response from the given request and update
+ all related objects accordingly.
- * ``view(response)`` - open the given response in your local web browser, for
- inspection. This will add a `\ tag`_ to the response body in order
- for external links (such as images and style sheets) to display properly.
- Note, however, that this will create a temporary file in your computer,
- which won't be removed automatically.
+- ``view(response)`` - open the given response in your local web browser, for
+ inspection. This will add a `\ tag`_ to the response body in order
+ for external links (such as images and style sheets) to display properly.
+ Note, however, that this will create a temporary file in your computer,
+ which won't be removed automatically.
.. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
@@ -122,21 +123,21 @@ content).
Those objects are:
- * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
+- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
- * ``spider`` - the Spider which is known to handle the URL, or a
- :class:`~scrapy.spiders.Spider` object if there is no spider found for
- the current URL
+- ``spider`` - the Spider which is known to handle the URL, or a
+ :class:`~scrapy.spiders.Spider` object if there is no spider found for the
+ current URL
- * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
- page. You can modify this request using :meth:`~scrapy.http.Request.replace`
- or fetch a new request (without leaving the shell) using the ``fetch``
- shortcut.
+- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
+ page. You can modify this request using
+ :meth:`~scrapy.http.Request.replace` or fetch a new request (without
+ leaving the shell) using the ``fetch`` shortcut.
- * ``response`` - a :class:`~scrapy.http.Response` object containing the last
- fetched page
+- ``response`` - a :class:`~scrapy.http.Response` object containing the last
+ fetched page
- * ``settings`` - the current :ref:`Scrapy settings `
+- ``settings`` - the current :ref:`Scrapy settings `
Example of shell session
========================
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 255ba9d3f..3d838fb63 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -384,6 +384,11 @@ bytes_received
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
+ Handlers for this signal can stop the download of a response while it
+ is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
+ exception. Please refer to the :ref:`topics-stop-response-download` topic
+ for additional information and examples.
+
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
@@ -395,11 +400,36 @@ bytes_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
-.. note:: Handlers of this signal can stop the download of a response while it
+headers_received
+~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.5
+
+.. signal:: headers_received
+.. function:: headers_received(headers, request, spider)
+
+ Sent by the HTTP 1.1 and S3 download handlers when the response headers are
+ available for a given request, before downloading any additional content.
+
+ Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
+ This signal does not support returning deferreds from its handlers.
+
+ :param headers: the headers received by the download handler
+ :type headers: :class:`scrapy.http.headers.Headers` object
+
+ :param body_length: expected size of the response body, in bytes
+ :type body_length: `int`
+
+ :param request: the request that generated the download
+ :type request: :class:`~scrapy.http.Request` object
+
+ :param spider: the spider associated with the response
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
Response signals
----------------
@@ -423,6 +453,11 @@ response_received
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
+.. note:: The ``request`` argument might not contain the original request that
+ reached the downloader, if a :ref:`topics-downloader-middleware` modifies
+ the :class:`~scrapy.http.Response` object and sets a specific ``request``
+ attribute.
+
response_downloaded
~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index c6cbdba76..11bbbb58d 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings `.
.. method:: process_start_requests(start_requests, spider)
- .. versionadded:: 0.15
-
This method is called with the start requests of the spider, and works
similarly to the :meth:`process_spider_output` method, except that it
doesn't have a response associated and must return only requests (not
@@ -255,7 +253,8 @@ this::
The ``handle_httpstatus_list`` key of :attr:`Request.meta
` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
-to ``True`` if you want to allow any response code for a request.
+to ``True`` if you want to allow any response code for a request, and ``False`` to
+disable the effects of the ``handle_httpstatus_all`` key.
Keep in mind, however, that it's usually a bad idea to handle non-200
responses, unless you really know what you're doing.
@@ -341,8 +340,6 @@ RefererMiddleware settings
REFERER_ENABLED
^^^^^^^^^^^^^^^
-.. versionadded:: 0.15
-
Default: ``True``
Whether to enable referer middleware.
@@ -352,8 +349,6 @@ Whether to enable referer middleware.
REFERRER_POLICY
^^^^^^^^^^^^^^^
-.. versionadded:: 1.4
-
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d4d6e2ea0..a3e9f410f 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
- self.start_urls = ['http://www.example.com/categories/%s' % category]
+ self.start_urls = [f'http://www.example.com/categories/{category}']
# ...
The default `__init__` method will take any spider arguments
@@ -292,7 +292,15 @@ The above example can also be written as follows::
name = 'myspider'
def start_requests(self):
- yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
+ yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
+
+If you are :ref:`running Scrapy from a script `, you can
+specify spider arguments when calling
+:class:`CrawlerProcess.crawl ` or
+:class:`CrawlerRunner.crawl `::
+
+ process = CrawlerProcess()
+ process.crawl(MySpider, category="electronics")
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
@@ -360,9 +368,10 @@ CrawlSpider
This spider also exposes an overrideable method:
- .. method:: parse_start_url(response)
+ .. method:: parse_start_url(response, **kwargs)
- This method is called for the start_urls responses. It allows to parse
+ This method is called for each response produced for the URLs in
+ the spider's ``start_urls`` attribute. It allows to parse
the initial responses and must return either an
:ref:`item object `, a :class:`~scrapy.http.Request`
object, or an iterable containing any of them.
@@ -388,11 +397,6 @@ Crawling rules
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
- .. warning:: When writing crawl spider rules, avoid using ``parse`` as
- callback, since the :class:`CrawlSpider` uses the ``parse`` method
- itself to implement its logic. So if you override the ``parse`` method,
- the crawl spider will no longer work.
-
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
callback function.
@@ -418,6 +422,11 @@ Crawling rules
It receives a :class:`Twisted Failure `
instance as first parameter.
+
+.. warning:: Because of its internal implementation, you must explicitly set
+ callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+ unexpected behaviour can occur otherwise.
+
.. versionadded:: 2.0
The *errback* parameter.
@@ -451,6 +460,11 @@ Let's now take a look at an example CrawlSpider with rules::
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
+ url = response.xpath('//td[@id="additional_data"]/@href').get()
+ return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
+
+ def parse_additional_page(self, response, item):
+ item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
return item
@@ -544,6 +558,11 @@ XMLFeedSpider
those results. It must return a list of results (items or requests).
+.. warning:: Because of its internal implementation, you must explicitly set
+ callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+ unexpected behaviour can occur otherwise.
+
+
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 9802a34a2..832829b75 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -110,11 +110,10 @@ using the telnet console::
Execution engine status
time()-engine.start_time : 8.62972998619
- engine.has_capacity() : False
len(engine.downloader.active) : 16
engine.scraper.is_idle() : False
engine.spider.name : followall
- engine.spider_is_idle(engine.spider) : False
+ engine.spider_is_idle() : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 9acfc3b23..95a3f17d5 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -23,7 +23,7 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
- line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+ line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 227085f02..57643ea9a 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -1,7 +1,7 @@
.. _versioning:
============================
-Versioning and API Stability
+Versioning and API stability
============================
Versioning
@@ -34,7 +34,7 @@ For example:
production)
-API Stability
+API stability
=============
API stability was one of the major goals for the *1.0* release.
@@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
same way.
+.. _deprecation-policy:
+
+Deprecation policy
+==================
+
+We aim to maintain support for deprecated Scrapy features for at least 1 year.
+
+For example, if a feature is deprecated in a Scrapy version released on
+June 15th 2020, that feature should continue to work in versions released on
+June 14th 2021 or before that.
+
+Any new Scrapy release after a year *may* remove support for that deprecated
+feature.
+
+All deprecated features removed in a Scrapy release are explicitly mentioned in
+the :ref:`release notes `.
+
+
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index da7a0022b..a6472b1ba 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
- print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
+ print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args:
latency = float(request.args['latency'][0])
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 7554f7eec..f9fb70342 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
- super(QPSSpider, self).__init__(*a, **kw)
+ super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
@@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
- url += '?latency={0}'.format(self.latency)
+ url += f'?latency={self.latency}'
slots = int(self.slots)
if slots > 1:
- urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
+ urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else:
urls = [url]
diff --git a/pylintrc b/pylintrc
index 129c7bf7d..972bf99de 100644
--- a/pylintrc
+++ b/pylintrc
@@ -24,6 +24,7 @@ disable=abstract-method,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
+ consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
@@ -68,6 +69,7 @@ disable=abstract-method,
pointless-statement,
pointless-string-statement,
protected-access,
+ raise-missing-from,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
@@ -75,6 +77,7 @@ disable=abstract-method,
signature-differs,
singleton-comparison,
super-init-not-called,
+ super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
diff --git a/pytest.ini b/pytest.ini
index 663c5cc78..6de08c78d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +1,5 @@
[pytest]
+xfail_strict = true
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
@@ -17,26 +18,7 @@ addopts =
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
-twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
-flake8-max-line-length = 119
-flake8-ignore =
- W503
-
- # Exclude files that are meant to provide top-level imports
- # E402: Module level import not at top of file
- # F401: Module imported but unused
- scrapy/__init__.py E402
- scrapy/core/downloader/handlers/http.py F401
- scrapy/http/__init__.py F401
- scrapy/linkextractors/__init__.py E402 F401
- scrapy/selector/__init__.py F401
- scrapy/spiders/__init__.py E402 F401
-
- # Issues pending a review:
- scrapy/utils/http.py F403
- scrapy/utils/markup.py F403
- scrapy/utils/multipart.py F403
- scrapy/utils/url.py F403 F405
- tests/test_loader.py E741
+filterwarnings=
+ ignore::DeprecationWarning:twisted.web.test.test_webclient
diff --git a/scrapy/VERSION b/scrapy/VERSION
index ccbccc3dc..437459cd9 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.2.0
+2.5.0
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index f0259a9b7..8a8065bf2 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -22,14 +22,14 @@ __all__ = [
# Scrapy and Twisted versions
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
-if sys.version_info < (3, 5, 2):
- print("Scrapy %s requires Python 3.5.2" % __version__)
+if sys.version_info < (3, 6):
+ print("Scrapy %s requires Python 3.6+" % __version__)
sys.exit(1)
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index b189e016b..91482ce01 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
- if inspect.isclass(obj) and \
- issubclass(obj, ScrapyCommand) and \
- obj.__module__ == module.__name__ and \
- not obj == ScrapyCommand:
+ if (
+ inspect.isclass(obj)
+ and issubclass(obj, ScrapyCommand)
+ and obj.__module__ == module.__name__
+ and not obj == ScrapyCommand
+ ):
yield obj
@@ -42,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
else:
- raise Exception("Invalid entry point %s" % entry_point.name)
+ raise Exception(f"Invalid entry point {entry_point.name}")
return cmds
@@ -65,11 +67,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
+ version = scrapy.__version__
if inproject:
- print("Scrapy %s - project: %s\n" % (scrapy.__version__,
- settings['BOT_NAME']))
+ print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
else:
- print("Scrapy %s - no active project\n" % scrapy.__version__)
+ print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
@@ -79,7 +81,7 @@ def _print_commands(settings, inproject):
print("Available commands:")
cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()):
- print(" %-13s %s" % (cmdname, cmdclass.short_desc()))
+ print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject:
print()
print(" [ more ] More commands available when run from project directory")
@@ -89,7 +91,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
- print("Unknown command: %s\n" % cmdname)
+ print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
@@ -131,7 +133,7 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
- parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
+ parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
@@ -153,7 +155,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts):
if opts.profile:
- sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
+ sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index ab850dcb3..6e77551c6 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -3,6 +3,8 @@ Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
+from typing import Any, Dict
+
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@@ -15,7 +17,7 @@ class ScrapyCommand:
crawler_process = None
# default settings to be used for this command instead of global defaults
- default_settings = {}
+ default_settings: Dict[str, Any] = {}
exitcode = 0
@@ -61,7 +63,7 @@ class ScrapyCommand:
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
- help="log level (default: %s)" % self.settings['LOG_LEVEL'])
+ help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
@@ -108,16 +110,18 @@ class ScrapyCommand:
class BaseRunSpiderCommand(ScrapyCommand):
"""
- Common class used to share functionality between the crawl and runspider commands
+ Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
- help="dump scraped items into FILE (use - for stdout)")
+ help="append scraped items to the end of FILE (use - for stdout)")
+ parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
+ help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
- help="format to use for dumping items with -o")
+ help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@@ -125,6 +129,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
- if opts.output:
- feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+ if opts.output or opts.overwrite_output:
+ feeds = feed_process_params_from_cli(
+ self.settings,
+ opts.output,
+ opts.output_format,
+ opts.overwrite_output,
+ )
self.settings.set('FEEDS', feeds, priority='cmdline')
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index c9f3b38e0..6bdf9eae0 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
- url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
+ url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 9d4437a47..ae21d86e6 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else ""
writeln(self.separator2)
- writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
+ writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
writeln()
infos = []
@@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED")
failed, errored = map(len, (self.failures, self.errors))
if failed:
- infos.append("failures=%d" % failed)
+ infos.append(f"failures={failed}")
if errored:
- infos.append("errors=%d" % errored)
+ infos.append(f"errors={errored}")
else:
write("OK")
if infos:
- writeln(" (%s)" % (", ".join(infos),))
+ writeln(f" ({', '.join(infos)})")
else:
write("\n")
@@ -78,19 +78,19 @@ class Command(ScrapyCommand):
elif tested_methods:
self.crawler_process.crawl(spidercls)
- # start checks
- if opts.list:
- for spider, methods in sorted(contract_reqs.items()):
- if not methods and not opts.verbose:
- continue
- print(spider)
- for method in sorted(methods):
- print(' * %s' % method)
- else:
- start = time.time()
- self.crawler_process.start()
- stop = time.time()
+ # start checks
+ if opts.list:
+ for spider, methods in sorted(contract_reqs.items()):
+ if not methods and not opts.verbose:
+ continue
+ print(spider)
+ for method in sorted(methods):
+ print(f' * {method}')
+ else:
+ start = time.time()
+ self.crawler_process.start()
+ stop = time.time()
- result.printErrors()
- result.printSummary(start, stop)
- self.exitcode = int(not result.wasSuccessful())
+ result.printErrors()
+ result.printSummary(start, stop)
+ self.exitcode = int(not result.wasSuccessful())
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index e1724c1e6..0f2a21b85 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
- raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
+ raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
@@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
else:
self.crawler_process.start()
- if self.crawler_process.bootstrap_failed or \
- (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+ if (
+ self.crawler_process.bootstrap_failed
+ or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
+ ):
self.exitcode = 1
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 25d843a53..177b20143 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
- return self._err("Spider not found: %s" % args[0])
+ return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
- self.exitcode = os.system('%s "%s"' % (editor, sfile))
+ self.exitcode = os.system(f'{editor} "{sfile}"')
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 063195f50..95f87e8c3 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -19,8 +19,10 @@ class Command(ScrapyCommand):
return "Fetch a URL using the Scrapy downloader"
def long_desc(self):
- return "Fetch a URL using the Scrapy downloader and print its content " \
- "to stdout. You may want to use --nolog to disable logging"
+ return (
+ "Fetch a URL using the Scrapy downloader and print its content"
+ " to stdout. You may want to use --nolog to disable logging"
+ )
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index abf3b7a5c..5f44daa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -66,31 +66,25 @@ class Command(ScrapyCommand):
print("Cannot create a spider with the same name as your project")
return
- try:
- spidercls = self.crawler_process.spider_loader.load(name)
- except KeyError:
- pass
- else:
- # if spider already exists and not --force then halt
- if not opts.force:
- print("Spider %r already exists in module:" % name)
- print(" %s" % spidercls.__module__)
- return
+ if not opts.force and self._spider_exists(name):
+ return
+
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
if opts.edit:
- self.exitcode = os.system('scrapy edit "%s"' % name)
+ self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
+ capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
- 'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
+ 'classname': f'{capitalized_module}Spider'
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@@ -98,29 +92,58 @@ class Command(ScrapyCommand):
else:
spiders_module = None
spiders_dir = "."
- spider_file = "%s.py" % join(spiders_dir, module)
+ spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
- print("Created spider %r using template %r "
- % (name, template_name), end=('' if spiders_module else '\n'))
+ print(f"Created spider {name!r} using template {template_name!r} ",
+ end=('' if spiders_module else '\n'))
if spiders_module:
- print("in module:\n %s.%s" % (spiders_module.__name__, module))
+ print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
- template_file = join(self.templates_dir, '%s.tmpl' % template)
+ template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
return template_file
- print("Unable to find template: %s\n" % template)
+ print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
- print(" %s" % splitext(filename)[0])
+ print(f" {splitext(filename)[0]}")
+
+ def _spider_exists(self, name):
+ if not self.settings.get('NEWSPIDER_MODULE'):
+ # if run as a standalone command and file with same filename already exists
+ if exists(name + ".py"):
+ print(f"{abspath(name + '.py')} already exists")
+ return True
+ return False
+
+ try:
+ spidercls = self.crawler_process.spider_loader.load(name)
+ except KeyError:
+ pass
+ else:
+ # if spider with same name exists
+ print(f"Spider {name!r} already exists in module:")
+ print(f" {spidercls.__module__}")
+ return True
+
+ # a file with the same name exists in the target directory
+ spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
+ spiders_dir = dirname(spiders_module.__file__)
+ spiders_dir_abs = abspath(spiders_dir)
+ if exists(join(spiders_dir_abs, name + ".py")):
+ print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
+ return True
+
+ return False
@property
def templates_dir(self):
- _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
- join(scrapy.__path__[0], 'templates')
- return join(_templates_base_dir, 'spiders')
+ return join(
+ self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+ 'spiders'
+ )
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 8b7fa8b58..52118db1b 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,26 +1,26 @@
import json
import logging
+from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.utils import display
-from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
+
logger = logging.getLogger(__name__)
-class Command(ScrapyCommand):
-
+class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
- items = {}
- requests = {}
+ items: Dict[int, list] = {}
+ requests: Dict[int, list] = {}
first_response = None
@@ -31,11 +31,9 @@ class Command(ScrapyCommand):
return "Parse URL (using its spider) and print the results"
def add_options(self, parser):
- ScrapyCommand.add_options(self, parser)
+ BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
- parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
- help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
@@ -100,13 +98,13 @@ class Command(ScrapyCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
- print('\n>>> DEPTH LEVEL: %s <<<' % level)
+ print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
- print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
+ print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
@@ -200,12 +198,15 @@ class Command(ScrapyCommand):
self.add_items(depth, items)
self.add_requests(depth, requests)
+ scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
- return requests
+ scraped_data += requests
+
+ return scraped_data
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
@@ -221,18 +222,11 @@ class Command(ScrapyCommand):
return request
def process_options(self, args, opts):
- ScrapyCommand.process_options(self, args, opts)
+ BaseRunSpiderCommand.process_options(self, args, opts)
- self.process_spider_arguments(opts)
self.process_request_meta(opts)
self.process_request_cb_kwargs(opts)
- def process_spider_arguments(self, opts):
- try:
- opts.spargs = arglist_to_dict(opts.spargs)
- except ValueError:
- raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
-
def process_request_meta(self, opts):
if opts.meta:
try:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index befee021b..b957c29fb 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -11,8 +11,8 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
- if fext != '.py':
- raise ValueError("Not a Python source file: %s" % abspath)
+ if fext not in ('.py', '.pyw'):
+ raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
@@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand):
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
- raise UsageError("File not found: %s\n" % filename)
+ raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
- raise UsageError("Unable to load %r: %s\n" % (filename, e))
+ raise UsageError(f"Unable to load {filename!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
- raise UsageError("No spider found in file: %s\n" % filename)
+ raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs)
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 852281959..1d73fa0cb 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
import re
import os
-import stat
import string
from importlib import import_module
from os.path import join, exists, abspath
from shutil import ignore_patterns, move, copy2, copystat
+from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
('${project_name}', 'middlewares.py.tmpl'),
)
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+ current_permissions = os.stat(path).st_mode
+ os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
class Command(ScrapyCommand):
@@ -47,7 +52,7 @@ class Command(ScrapyCommand):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
- print('Error: Module %r already exists' % project_name)
+ print(f'Error: Module {project_name!r} already exists')
else:
return True
return False
@@ -78,30 +83,10 @@ class Command(ScrapyCommand):
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
+ _make_writable(dstname)
+
copystat(src, dst)
- self._set_rw_permissions(dst)
-
- def _set_rw_permissions(self, path):
- """
- Sets permissions of a directory tree to +rw and +rwx for folders.
- This is necessary if the start template files come without write
- permissions.
- """
- mode_rw = (stat.S_IRUSR
- | stat.S_IWUSR
- | stat.S_IRGRP
- | stat.S_IROTH)
-
- mode_x = (stat.S_IXUSR
- | stat.S_IXGRP
- | stat.S_IXOTH)
-
- os.chmod(path, mode_rw | mode_x)
- for root, dirs, files in os.walk(path):
- for dir in dirs:
- os.chmod(join(root, dir), mode_rw | mode_x)
- for file in files:
- os.chmod(join(root, file), mode_rw)
+ _make_writable(dst)
def run(self, args, opts):
if len(args) not in (1, 2):
@@ -115,7 +100,7 @@ class Command(ScrapyCommand):
if exists(join(project_dir, 'scrapy.cfg')):
self.exitcode = 1
- print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
+ print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
return
if not self._is_valid_name(project_name):
@@ -128,15 +113,16 @@ class Command(ScrapyCommand):
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
- print("New Scrapy project '%s', using template directory '%s', "
- "created in:" % (project_name, self.templates_dir))
- print(" %s\n" % abspath(project_dir))
+ print(f"New Scrapy project '{project_name}', using template directory "
+ f"'{self.templates_dir}', created in:")
+ print(f" {abspath(project_dir)}\n")
print("You can start your first spider with:")
- print(" cd %s" % project_dir)
+ print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
- _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
- join(scrapy.__path__[0], 'templates')
- return join(_templates_base_dir, 'project')
+ return join(
+ self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
+ 'project'
+ )
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index d0ea72a67..1237610cb 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -23,8 +23,7 @@ class Command(ScrapyCommand):
if opts.verbose:
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)
- patt = "%-{}s : %s".format(width)
for name, version in versions:
- print(patt % (name, version))
+ print(f"{name:<{width}} : {version}")
else:
- print("Scrapy %s" % scrapy.__version__)
+ print(f"Scrapy {scrapy.__version__}")
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 41e77ba3b..c8f873334 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -8,11 +8,10 @@ class Command(fetch.Command):
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
- return "Fetch a URL using the Scrapy downloader and show its " \
- "contents in a browser"
+ return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
def add_options(self, parser):
- super(Command, self).add_options(parser)
+ super().add_options(parser)
parser.remove_option("--headers")
def _print_response(self, response, opts):
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 5af3831a2..b47e55092 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,16 +1,77 @@
-import sys
import re
+import sys
from functools import wraps
from inspect import getmembers
+from typing import Dict
from unittest import TestCase
from scrapy.http import Request
-from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
+from scrapy.utils.spider import iterate_spider_output
+
+
+class Contract:
+ """ Abstract class for contracts """
+ request_cls = None
+
+ def __init__(self, method, *args):
+ self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
+ self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
+ self.args = args
+
+ def add_pre_hook(self, request, results):
+ if hasattr(self, 'pre_process'):
+ cb = request.callback
+
+ @wraps(cb)
+ def wrapper(response, **cb_kwargs):
+ try:
+ results.startTest(self.testcase_pre)
+ self.pre_process(response)
+ results.stopTest(self.testcase_pre)
+ except AssertionError:
+ results.addFailure(self.testcase_pre, sys.exc_info())
+ except Exception:
+ results.addError(self.testcase_pre, sys.exc_info())
+ else:
+ results.addSuccess(self.testcase_pre)
+ finally:
+ return list(iterate_spider_output(cb(response, **cb_kwargs)))
+
+ request.callback = wrapper
+
+ return request
+
+ def add_post_hook(self, request, results):
+ if hasattr(self, 'post_process'):
+ cb = request.callback
+
+ @wraps(cb)
+ def wrapper(response, **cb_kwargs):
+ output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+ try:
+ results.startTest(self.testcase_post)
+ self.post_process(output)
+ results.stopTest(self.testcase_post)
+ except AssertionError:
+ results.addFailure(self.testcase_post, sys.exc_info())
+ except Exception:
+ results.addError(self.testcase_post, sys.exc_info())
+ else:
+ results.addSuccess(self.testcase_post)
+ finally:
+ return output
+
+ request.callback = wrapper
+
+ return request
+
+ def adjust_request_args(self, args):
+ return args
class ContractsManager:
- contracts = {}
+ contracts: Dict[str, Contract] = {}
def __init__(self, contracts):
for contract in contracts:
@@ -107,73 +168,13 @@ class ContractsManager:
request.errback = eb_wrapper
-class Contract:
- """ Abstract class for contracts """
- request_cls = None
-
- def __init__(self, method, *args):
- self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
- self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name)
- self.args = args
-
- def add_pre_hook(self, request, results):
- if hasattr(self, 'pre_process'):
- cb = request.callback
-
- @wraps(cb)
- def wrapper(response, **cb_kwargs):
- try:
- results.startTest(self.testcase_pre)
- self.pre_process(response)
- results.stopTest(self.testcase_pre)
- except AssertionError:
- results.addFailure(self.testcase_pre, sys.exc_info())
- except Exception:
- results.addError(self.testcase_pre, sys.exc_info())
- else:
- results.addSuccess(self.testcase_pre)
- finally:
- return list(iterate_spider_output(cb(response, **cb_kwargs)))
-
- request.callback = wrapper
-
- return request
-
- def add_post_hook(self, request, results):
- if hasattr(self, 'post_process'):
- cb = request.callback
-
- @wraps(cb)
- def wrapper(response, **cb_kwargs):
- output = list(iterate_spider_output(cb(response, **cb_kwargs)))
- try:
- results.startTest(self.testcase_post)
- self.post_process(output)
- results.stopTest(self.testcase_post)
- except AssertionError:
- results.addFailure(self.testcase_post, sys.exc_info())
- except Exception:
- results.addError(self.testcase_post, sys.exc_info())
- else:
- results.addSuccess(self.testcase_post)
- finally:
- return output
-
- request.callback = wrapper
-
- return request
-
- def adjust_request_args(self, args):
- return args
-
-
def _create_testcase(method, desc):
spider = method.__self__.name
class ContractTestCase(TestCase):
def __str__(_self):
- return "[%s] %s (%s)" % (spider, method.__name__, desc)
+ return f"[{spider}] {method.__name__} ({desc})"
- name = '%s_%s' % (spider, method.__name__)
+ name = f'{spider}_{method.__name__}'
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 34f0d36d4..9704f5253 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -56,12 +56,11 @@ class ReturnsContract(Contract):
}
def __init__(self, *args, **kwargs):
- super(ReturnsContract, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
raise ValueError(
- "Incorrect argument quantity: expected 1, 2 or 3, got %i"
- % len(self.args)
+ f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
)
self.obj_name = self.args[0] or None
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
@@ -88,10 +87,9 @@ class ReturnsContract(Contract):
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
- expected = '%s..%s' % (self.min_bound, self.max_bound)
+ expected = f'{self.min_bound}..{self.max_bound}'
- raise ContractFail("Returned %s %s, expected %s" %
- (occurrences, self.obj_name, expected))
+ raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
class ScrapesContract(Contract):
@@ -106,5 +104,5 @@ class ScrapesContract(Contract):
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
- missing_str = ", ".join(missing)
- raise ContractFail("Missing fields: %s" % missing_str)
+ missing_fields = ", ".join(missing)
+ raise ContractFail(f"Missing fields: {missing_fields}")
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index dc5cf1ab8..4f7ab594f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -41,17 +41,17 @@ class Slot:
def __repr__(self):
cls_name = self.__class__.__name__
- return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % (
- cls_name, self.concurrency, self.delay, self.randomize_delay)
+ return (f"{cls_name}(concurrency={self.concurrency!r}, "
+ f"delay={self.delay:.2f}, "
+ f"randomize_delay={self.randomize_delay!r})")
def __str__(self):
return (
- "" % (
- self.concurrency, self.delay, self.randomize_delay,
- len(self.active), len(self.queue), len(self.transferring),
- datetime.fromtimestamp(self.lastseen).isoformat()
- )
+ f""
)
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..073ef16bf 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,10 +1,15 @@
+import warnings
+
from OpenSSL import SSL
+from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
+from zope.interface.verify import verifyObject
-from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
+from scrapy.utils.misc import create_instance, load_object
@implementer(IPolicyForHTTPS)
@@ -20,7 +25,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
@@ -45,7 +50,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
#
# * getattr() for `_ssl_method` attribute for context factories
- # not calling super(..., self).__init__
+ # not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
@@ -81,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
The default OpenSSL method is ``TLS_METHOD`` (also called
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
- def creatorForNetloc(self, hostname, port):
+ def creatorForNetloc(self, hostname, port):
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@@ -92,3 +97,49 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)
+
+
+@implementer(IPolicyForHTTPS)
+class AcceptableProtocolsContextFactory:
+ """Context factory to used to override the acceptable protocols
+ to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
+ negotiation.
+ """
+
+ def __init__(self, context_factory, acceptable_protocols):
+ verifyObject(IPolicyForHTTPS, context_factory)
+ self._wrapped_context_factory = context_factory
+ self._acceptable_protocols = acceptable_protocols
+
+ def creatorForNetloc(self, hostname, port):
+ options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
+ _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
+ return options
+
+
+def load_context_factory_from_settings(settings, crawler):
+ ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
+ context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ # try method-aware context factory
+ try:
+ context_factory = create_instance(
+ objcls=context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ method=ssl_method,
+ )
+ except TypeError:
+ # use context factory defaults
+ context_factory = create_instance(
+ objcls=context_factory_cls,
+ settings=settings,
+ crawler=crawler,
+ )
+ msg = """
+ '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
+ settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+ warnings.warn(msg)
+
+ return context_factory
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index e86680978..73aeb2352 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -71,8 +71,7 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
- raise NotSupported("Unsupported URL scheme '%s': %s" %
- (scheme, self._notconfigured[scheme]))
+ raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
@defer.inlineCallbacks
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 15de8cdbd..12ca7ecad 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
-from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
-from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
@@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
- self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
- self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
- # try method-aware context factory
- try:
- self._contextFactory = create_instance(
- objcls=self._contextFactoryClass,
- settings=settings,
- crawler=crawler,
- method=self._sslMethod,
- )
- except TypeError:
- # use context factory defaults
- self._contextFactory = create_instance(
- objcls=self._contextFactoryClass,
- settings=settings,
- crawler=crawler,
- )
- msg = """
- '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
- Please upgrade your context factory class to handle them or ignore them.""" % (
- settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
- warnings.warn(msg)
+ self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
@@ -121,12 +98,13 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
-
- _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})')
+ _truncatedLength = 1000
+ _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})'
+ _responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
- super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
+ super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@@ -167,10 +145,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
else:
- extra = rcvd_bytes[:32]
+ extra = rcvd_bytes[:self._truncatedLength]
self._tunnelReadyDeferred.errback(
- TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
- self._host, self._port, extra)))
+ TunnelError('Could not open CONNECT tunnel with proxy '
+ f'{self._host}:{self._port} [{extra!r}]')
+ )
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@@ -178,7 +157,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
- connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
+ connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@@ -215,7 +194,7 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
- super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
+ super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@@ -235,7 +214,7 @@ class TunnelingAgent(Agent):
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
- return super(TunnelingAgent, self)._requestWithEndpoint(
+ return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
@@ -249,7 +228,7 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
- super(ScrapyProxyAgent, self).__init__(
+ super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
@@ -302,11 +281,14 @@ class ScrapyAgent:
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
- warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
- "If you use Crawlera, it doesn't require this mode anymore, "
- "so you should update scrapy-crawlera to 1.3.0+ "
- "and remove '?noconnect' from the Crawlera URL.",
- ScrapyDeprecationWarning)
+ warnings.warn(
+ "Using HTTPS proxies in the noconnect mode is deprecated. "
+ "If you use Zyte Smart Proxy Manager, it doesn't require "
+ "this mode anymore, so you should update scrapy-crawlera "
+ "to scrapy-zyte-smartproxy and remove '?noconnect' "
+ "from the Zyte Smart Proxy Manager URL.",
+ ScrapyDeprecationWarning,
+ )
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
@@ -375,13 +357,44 @@ class ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
- raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
+ raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
return result
+ @staticmethod
+ def _headers_from_twisted_response(response):
+ headers = Headers()
+ if response.length != UNKNOWN_LENGTH:
+ headers[b'Content-Length'] = str(response.length).encode()
+ headers.update(response.headers.getAllRawHeaders())
+ return headers
+
def _cb_bodyready(self, txresponse, request):
+ headers_received_result = self._crawler.signals.send_catch_log(
+ signal=signals.headers_received,
+ headers=self._headers_from_twisted_response(txresponse),
+ body_length=txresponse.length,
+ request=request,
+ spider=self._crawler.spider,
+ )
+ for handler, result in headers_received_result:
+ if isinstance(result, Failure) and isinstance(result.value, StopDownload):
+ logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
+ {"request": request, "handler": handler.__qualname__})
+ txresponse._transport.stopProducing()
+ with suppress(AttributeError):
+ txresponse._transport._producer.loseConnection()
+ return {
+ "txresponse": txresponse,
+ "body": b"",
+ "flags": ["download_stopped"],
+ "certificate": None,
+ "ip_address": None,
+ "failure": result if result.value.fail else None,
+ }
+
# deliverBody hangs for responses without body
if txresponse.length == 0:
return {
@@ -398,13 +411,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
- error_msg = ("Cancelling download of %(url)s: expected response "
- "size (%(size)s) larger than download max size (%(maxsize)s).")
- error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+ warning_msg = ("Cancelling download of %(url)s: expected response "
+ "size (%(size)s) larger than download max size (%(maxsize)s).")
+ warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+
+ logger.warning(warning_msg, warning_args)
- logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
- raise defer.CancelledError(error_msg % error_args)
+ raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@@ -434,8 +448,13 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
- headers = Headers(result["txresponse"].headers.getAllRawHeaders())
+ headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
+ try:
+ version = result["txresponse"].version
+ protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
+ except (AttributeError, TypeError, IndexError):
+ protocol = None
response = respcls(
url=url,
status=int(result["txresponse"].code),
@@ -444,6 +463,7 @@ class ScrapyAgent:
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
+ protocol=protocol,
)
if result.get("failure"):
result["failure"].value.response = response
@@ -522,16 +542,17 @@ class _ResponseReader(protocol.Protocol):
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
+ self.transport.stopProducing()
self.transport._producer.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
- logger.error("Received (%(bytes)s) bytes larger than download "
- "max size (%(maxsize)s) in request %(request)s.",
- {'bytes': self._bytes_received,
- 'maxsize': self._maxsize,
- 'request': self._request})
+ logger.warning("Received (%(bytes)s) bytes larger than download "
+ "max size (%(maxsize)s) in request %(request)s.",
+ {'bytes': self._bytes_received,
+ 'maxsize': self._maxsize,
+ 'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
new file mode 100644
index 000000000..7bb88a193
--- /dev/null
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -0,0 +1,129 @@
+import warnings
+from time import time
+from typing import Optional, Type, TypeVar
+from urllib.parse import urldefrag
+
+from twisted.internet.base import DelayedCall
+from twisted.internet.defer import Deferred
+from twisted.internet.error import TimeoutError
+from twisted.web.client import URI
+
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
+from scrapy.core.downloader.webclient import _parse
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
+from scrapy.crawler import Crawler
+from scrapy.http import Request, Response
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
+from scrapy.utils.python import to_bytes
+
+
+H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
+
+
+class H2DownloadHandler:
+ def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
+ self._crawler = crawler
+
+ from twisted.internet import reactor
+ self._pool = H2ConnectionPool(reactor, settings)
+ self._context_factory = load_context_factory_from_settings(settings, crawler)
+
+ @classmethod
+ def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
+ return cls(crawler.settings, crawler)
+
+ def download_request(self, request: Request, spider: Spider) -> Deferred:
+ agent = ScrapyH2Agent(
+ context_factory=self._context_factory,
+ pool=self._pool,
+ crawler=self._crawler,
+ )
+ return agent.download_request(request, spider)
+
+ def close(self) -> None:
+ self._pool.close_connections()
+
+
+class ScrapyH2Agent:
+ _Agent = H2Agent
+ _ProxyAgent = ScrapyProxyH2Agent
+
+ def __init__(
+ self, context_factory,
+ pool: H2ConnectionPool,
+ connect_timeout: int = 10,
+ bind_address: Optional[bytes] = None,
+ crawler: Optional[Crawler] = None,
+ ) -> None:
+ self._context_factory = context_factory
+ self._connect_timeout = connect_timeout
+ self._bind_address = bind_address
+ self._pool = pool
+ self._crawler = crawler
+
+ def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
+ from twisted.internet import reactor
+ bind_address = request.meta.get('bindaddress') or self._bind_address
+ proxy = request.meta.get('proxy')
+ if proxy:
+ _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
+ scheme = _parse(request.url)[0]
+ proxy_host = proxy_host.decode()
+ omit_connect_tunnel = b'noconnect' in proxy_params
+ if omit_connect_tunnel:
+ warnings.warn(
+ "Using HTTPS proxies in the noconnect mode is not "
+ "supported by the downloader handler. If you use Zyte "
+ "Smart Proxy Manager, it doesn't require this mode "
+ "anymore, so you should update scrapy-crawlera to "
+ "scrapy-zyte-smartproxy and remove '?noconnect' from the "
+ "Zyte Smart Proxy Manager URL."
+ )
+
+ if scheme == b'https' and not omit_connect_tunnel:
+ # ToDo
+ raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
+ return self._ProxyAgent(
+ reactor=reactor,
+ context_factory=self._context_factory,
+ proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
+ connect_timeout=timeout,
+ bind_address=bind_address,
+ pool=self._pool,
+ )
+
+ return self._Agent(
+ reactor=reactor,
+ context_factory=self._context_factory,
+ connect_timeout=timeout,
+ bind_address=bind_address,
+ pool=self._pool,
+ )
+
+ def download_request(self, request: Request, spider: Spider) -> Deferred:
+ from twisted.internet import reactor
+ timeout = request.meta.get('download_timeout') or self._connect_timeout
+ agent = self._get_agent(request, timeout)
+
+ start_time = time()
+ d = agent.request(request, spider)
+ d.addCallback(self._cb_latency, request, start_time)
+
+ timeout_cl = reactor.callLater(timeout, d.cancel)
+ d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
+ return d
+
+ @staticmethod
+ def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
+ request.meta['download_latency'] = time() - start_time
+ return response
+
+ @staticmethod
+ def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
+ if timeout_cl.active():
+ timeout_cl.cancel()
+ return response
+
+ url = urldefrag(request.url)[0]
+ raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 8f63ad974..1966570d4 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,41 +2,20 @@ from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
-def _get_boto_connection():
- from boto.s3.connection import S3Connection
-
- class _v19_S3Connection(S3Connection):
- """A dummy S3Connection wrapper that doesn't do any synchronous download"""
- def _mexe(self, method, bucket, key, headers, *args, **kwargs):
- return headers
-
- class _v20_S3Connection(S3Connection):
- """A dummy S3Connection wrapper that doesn't do any synchronous download"""
- def _mexe(self, http_request, *args, **kwargs):
- http_request.authorize(connection=self)
- return http_request.headers
-
- try:
- import boto.auth # noqa: F401
- except ImportError:
- _S3Connection = _v19_S3Connection
- else:
- _S3Connection = _v20_S3Connection
-
- return _S3Connection
-
-
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
+ if not is_botocore_available():
+ raise NotConfigured('missing botocore library')
+
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
@@ -51,23 +30,15 @@ class S3DownloadHandler:
self.anon = kw.get('anon')
self._signer = None
- if is_botocore():
- import botocore.auth
- import botocore.credentials
- kw.pop('anon', None)
- if kw:
- raise TypeError('Unexpected keyword arguments: %s' % kw)
- if not self.anon:
- SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
- self._signer = SignerCls(botocore.credentials.Credentials(
- aws_access_key_id, aws_secret_access_key))
- else:
- _S3Connection = _get_boto_connection()
- try:
- self.conn = _S3Connection(
- aws_access_key_id, aws_secret_access_key, **kw)
- except Exception as ex:
- raise NotConfigured(str(ex))
+ import botocore.auth
+ import botocore.credentials
+ kw.pop('anon', None)
+ if kw:
+ raise TypeError(f'Unexpected keyword arguments: {kw}')
+ if not self.anon:
+ SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+ self._signer = SignerCls(botocore.credentials.Credentials(
+ aws_access_key_id, aws_secret_access_key))
_http_handler = create_instance(
objcls=httpdownloadhandler,
@@ -85,14 +56,14 @@ class S3DownloadHandler:
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
- url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
+ url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
- url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
+ url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
headers=request.headers.to_unicode_dict(),
data=request.body)
self._signer.add_auth(awsrequest)
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 4c2eea522..a5619d8a4 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
-from twisted.internet import defer
+from typing import Callable, Union
+from twisted.internet import defer
+from twisted.python.failure import Failure
+
+from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
@@ -29,22 +33,22 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
- def download(self, download_func, request, spider):
+ def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
- def process_request(request):
+ def process_request(request: Request):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
- "Middleware %s.process_request must return None, Response or Request, got %s"
- % (method.__self__.__class__.__name__, response.__class__.__name__)
+ f"Middleware {method.__qualname__} must return None, Response or "
+ f"Request, got {response.__class__.__name__}"
)
if response:
return response
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
- def process_response(response):
+ def process_response(response: Union[Response, Request]):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
@@ -54,22 +58,22 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
- "Middleware %s.process_response must return Response or Request, got %s"
- % (method.__self__.__class__.__name__, type(response))
+ f"Middleware {method.__qualname__} must return Response or Request, "
+ f"got {type(response)}"
)
if isinstance(response, Request):
return response
return response
@defer.inlineCallbacks
- def process_exception(failure):
+ def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
- "Middleware %s.process_exception must return None, Response or Request, got %s"
- % (method.__self__.__class__.__name__, type(response))
+ f"Middleware {method.__qualname__} must return None, Response or "
+ f"Request, got {type(response)}"
)
if response:
return response
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index e43a3c83e..2b8990b75 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet.ssl import AcceptableCiphers
-from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@@ -28,13 +27,6 @@ openssl_methods = {
}
-if twisted_version < (17, 0, 0):
- from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
-else:
- def set_tlsext_host_name(connection, hostNameBytes):
- connection.set_tlsext_host_name(hostNameBytes)
-
-
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
@@ -47,26 +39,19 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
"""
def __init__(self, hostname, ctx, verbose_logging=False):
- super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+ super().__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL.SSL_CB_HANDSHAKE_START:
- set_tlsext_host_name(connection, self._hostnameBytes)
+ connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
- if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
- if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
- logger.debug('SSL connection to %s using protocol %s, cipher %s',
- self._hostnameASCII,
- connection.get_protocol_version_name(),
- connection.get_cipher_name(),
- )
- else:
- logger.debug('SSL connection to %s using cipher %s',
- self._hostnameASCII,
- connection.get_cipher_name(),
- )
+ logger.debug('SSL connection to %s using protocol %s, cipher %s',
+ self._hostnameASCII,
+ connection.get_protocol_version_name(),
+ connection.get_cipher_name(),
+ )
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 8b6f87c3f..915cb5fe3 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -2,13 +2,13 @@ import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
-from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes
@@ -91,22 +91,38 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
- defer.TimeoutError("Getting %s took longer than %s seconds."
- % (self.factory.url, self.factory.timeout)))
+ defer.TimeoutError(f"Getting {self.factory.url} took longer "
+ f"than {self.factory.timeout} seconds."))
-class ScrapyHTTPClientFactory(HTTPClientFactory):
- """Scrapy implementation of the HTTPClientFactory overwriting the
- setUrl method to make use of our Url object that cache the parse
- result.
- """
+# This class used to inherit from Twisted’s
+# twisted.web.client.HTTPClientFactory. When that class was deprecated in
+# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
+# non-overriden code into this class.
+class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
+
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
+ def _build_response(self, body, request):
+ request.meta['download_latency'] = self.headers_time - self.start_time
+ status = int(self.status)
+ headers = Headers(self.response_headers)
+ respcls = responsetypes.from_args(headers=headers, url=self._url)
+ return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
+
+ def _set_connection_attributes(self, request):
+ parsed = urlparse_cached(request)
+ self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+ proxy = request.meta.get('proxy')
+ if proxy:
+ self.scheme, _, self.host, self.port, _ = _parse(proxy)
+ self.path = self.url
+
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
@@ -141,21 +157,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
elif self.method == b'POST':
self.headers['Content-Length'] = 0
- def _build_response(self, body, request):
- request.meta['download_latency'] = self.headers_time - self.start_time
- status = int(self.status)
- headers = Headers(self.response_headers)
- respcls = responsetypes.from_args(headers=headers, url=self._url)
- return respcls(url=self._url, status=status, headers=headers, body=body)
+ def __repr__(self):
+ return f"<{self.__class__.__name__}: {self.url}>"
- def _set_connection_attributes(self, request):
- parsed = urlparse_cached(request)
- self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
- proxy = request.meta.get('proxy')
- if proxy:
- self.scheme, _, self.host, self.port, _ = _parse(proxy)
- self.path = self.url
+ def _cancelTimeout(self, result, timeoutCall):
+ if timeoutCall.active():
+ timeoutCall.cancel()
+ return result
+
+ def buildProtocol(self, addr):
+ p = ClientFactory.buildProtocol(self, addr)
+ p.followRedirect = self.followRedirect
+ p.afterFoundGet = self.afterFoundGet
+ if self.timeout:
+ timeoutCall = reactor.callLater(self.timeout, p.timeout)
+ self.deferred.addBoth(self._cancelTimeout, timeoutCall)
+ return p
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
+
+ def gotStatus(self, version, status, message):
+ """
+ Set the status of the request on us.
+ @param version: The HTTP version.
+ @type version: L{bytes}
+ @param status: The HTTP status code, an integer represented as a
+ bytestring.
+ @type status: L{bytes}
+ @param message: The HTTP status message.
+ @type message: L{bytes}
+ """
+ self.version, self.status, self.message = version, status, message
+
+ def page(self, page):
+ if self.waiting:
+ self.waiting = 0
+ self.deferred.callback(page)
+
+ def noPage(self, reason):
+ if self.waiting:
+ self.waiting = 0
+ self.deferred.errback(reason)
+
+ def clientConnectionFailed(self, _, reason):
+ """
+ When a connection attempt fails, the request cannot be issued. If no
+ result has yet been provided to the result Deferred, provide the
+ connection failure reason as an error result.
+ """
+ if self.waiting:
+ self.waiting = 0
+ # If the connection attempt failed, there is nothing more to
+ # disconnect, so just fire that Deferred now.
+ self._disconnectedDeferred.callback(None)
+ self.deferred.errback(reason)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index de0da4b70..dd3225082 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -1,51 +1,62 @@
"""
-This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
+This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
For more information see docs/topics/architecture.rst
"""
import logging
+import warnings
from time import time
+from typing import Callable, Iterable, Iterator, Optional, Set, Union
-from twisted.internet import defer, task
+from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Response, Request
-from scrapy.utils.misc import load_object
-from scrapy.utils.reactor import CallLaterOnce
+from scrapy.settings import BaseSettings
+from scrapy.spiders import Spider
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.reactor import CallLaterOnce
+
logger = logging.getLogger(__name__)
class Slot:
-
- def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
- self.closing = False
- self.inprogress = set() # requests in progress
- self.start_requests = iter(start_requests)
+ def __init__(
+ self,
+ start_requests: Iterable,
+ close_if_idle: bool,
+ nextcall: CallLaterOnce,
+ scheduler,
+ ) -> None:
+ self.closing: Optional[Deferred] = None
+ self.inprogress: Set[Request] = set()
+ self.start_requests: Optional[Iterator] = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
self.scheduler = scheduler
- self.heartbeat = task.LoopingCall(nextcall.schedule)
+ self.heartbeat = LoopingCall(nextcall.schedule)
- def add_request(self, request):
+ def add_request(self, request: Request) -> None:
self.inprogress.add(request)
- def remove_request(self, request):
+ def remove_request(self, request: Request) -> None:
self.inprogress.remove(request)
self._maybe_fire_closing()
- def close(self):
- self.closing = defer.Deferred()
+ def close(self) -> Deferred:
+ self.closing = Deferred()
self._maybe_fire_closing()
return self.closing
- def _maybe_fire_closing(self):
- if self.closing and not self.inprogress:
+ def _maybe_fire_closing(self) -> None:
+ if self.closing is not None and not self.inprogress:
if self.nextcall:
self.nextcall.cancel()
if self.heartbeat.running:
@@ -54,203 +65,236 @@ class Slot:
class ExecutionEngine:
-
- def __init__(self, crawler, spider_closed_callback):
+ def __init__(self, crawler, spider_closed_callback: Callable) -> None:
self.crawler = crawler
self.settings = crawler.settings
self.signals = crawler.signals
self.logformatter = crawler.logformatter
- self.slot = None
- self.spider = None
+ self.slot: Optional[Slot] = None
+ self.spider: Optional[Spider] = None
self.running = False
self.paused = False
- self.scheduler_cls = load_object(self.settings['SCHEDULER'])
+ self.scheduler_cls = self._get_scheduler_class(crawler.settings)
downloader_cls = load_object(self.settings['DOWNLOADER'])
self.downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
- @defer.inlineCallbacks
- def start(self):
- """Start the execution engine"""
+ def _get_scheduler_class(self, settings: BaseSettings) -> type:
+ from scrapy.core.scheduler import BaseScheduler
+ scheduler_cls = load_object(settings["SCHEDULER"])
+ if not issubclass(scheduler_cls, BaseScheduler):
+ raise TypeError(
+ f"The provided scheduler class ({settings['SCHEDULER']})"
+ " does not fully implement the scheduler interface"
+ )
+ return scheduler_cls
+
+ @inlineCallbacks
+ def start(self) -> Deferred:
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
- self._closewait = defer.Deferred()
+ self._closewait = Deferred()
yield self._closewait
- def stop(self):
- """Stop the execution engine gracefully"""
+ def stop(self) -> Deferred:
+ """Gracefully stop the execution engine"""
+ @inlineCallbacks
+ def _finish_stopping_engine(_) -> Deferred:
+ yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
+ self._closewait.callback(None)
+
if not self.running:
raise RuntimeError("Engine not running")
+
self.running = False
- dfd = self._close_all_spiders()
- return dfd.addBoth(lambda _: self._finish_stopping_engine())
+ dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
+ return dfd.addBoth(_finish_stopping_engine)
- def close(self):
- """Close the execution engine gracefully.
-
- If it has already been started, stop it. In all cases, close all spiders
- and the downloader.
+ def close(self) -> Deferred:
+ """
+ Gracefully close the execution engine.
+ If it has already been started, stop it. In all cases, close the spider and the downloader.
"""
if self.running:
- # Will also close spiders and downloader
- return self.stop()
- elif self.open_spiders:
- # Will also close downloader
- return self._close_all_spiders()
- else:
- return defer.succeed(self.downloader.close())
+ return self.stop() # will also close spider and downloader
+ if self.spider is not None:
+ return self.close_spider(self.spider, reason="shutdown") # will also close downloader
+ return succeed(self.downloader.close())
- def pause(self):
- """Pause the execution engine"""
+ def pause(self) -> None:
self.paused = True
- def unpause(self):
- """Resume the execution engine"""
+ def unpause(self) -> None:
self.paused = False
- def _next_request(self, spider):
- slot = self.slot
- if not slot:
- return
+ def _next_request(self) -> None:
+ assert self.slot is not None # typing
+ assert self.spider is not None # typing
if self.paused:
- return
+ return None
- while not self._needs_backout(spider):
- if not self._next_request_from_scheduler(spider):
- break
+ while not self._needs_backout() and self._next_request_from_scheduler() is not None:
+ pass
- if slot.start_requests and not self._needs_backout(spider):
+ if self.slot.start_requests is not None and not self._needs_backout():
try:
- request = next(slot.start_requests)
+ request = next(self.slot.start_requests)
except StopIteration:
- slot.start_requests = None
+ self.slot.start_requests = None
except Exception:
- slot.start_requests = None
- logger.error('Error while obtaining start requests',
- exc_info=True, extra={'spider': spider})
+ self.slot.start_requests = None
+ logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
else:
- self.crawl(request, spider)
+ self.crawl(request)
- if self.spider_is_idle(spider) and slot.close_if_idle:
- self._spider_idle(spider)
+ if self.spider_is_idle() and self.slot.close_if_idle:
+ self._spider_idle()
- def _needs_backout(self, spider):
- slot = self.slot
- return not self.running \
- or slot.closing \
- or self.downloader.needs_backout() \
- or self.scraper.slot.needs_backout()
+ def _needs_backout(self) -> bool:
+ return (
+ not self.running
+ or self.slot.closing # type: ignore[union-attr]
+ or self.downloader.needs_backout()
+ or self.scraper.slot.needs_backout() # type: ignore[union-attr]
+ )
- def _next_request_from_scheduler(self, spider):
- slot = self.slot
- request = slot.scheduler.next_request()
- if not request:
- return
- d = self._download(request, spider)
- d.addBoth(self._handle_downloader_output, request, spider)
+ def _next_request_from_scheduler(self) -> Optional[Deferred]:
+ assert self.slot is not None # typing
+ assert self.spider is not None # typing
+
+ request = self.slot.scheduler.next_request()
+ if request is None:
+ return None
+
+ d = self._download(request, self.spider)
+ d.addBoth(self._handle_downloader_output, request)
d.addErrback(lambda f: logger.info('Error while handling downloader output',
exc_info=failure_to_exc_info(f),
- extra={'spider': spider}))
- d.addBoth(lambda _: slot.remove_request(request))
+ extra={'spider': self.spider}))
+ d.addBoth(lambda _: self.slot.remove_request(request))
d.addErrback(lambda f: logger.info('Error while removing request from slot',
exc_info=failure_to_exc_info(f),
- extra={'spider': spider}))
- d.addBoth(lambda _: slot.nextcall.schedule())
+ extra={'spider': self.spider}))
+ d.addBoth(lambda _: self.slot.nextcall.schedule())
d.addErrback(lambda f: logger.info('Error while scheduling new request',
exc_info=failure_to_exc_info(f),
- extra={'spider': spider}))
+ extra={'spider': self.spider}))
return d
- def _handle_downloader_output(self, response, request, spider):
- if not isinstance(response, (Request, Response, Failure)):
- raise TypeError(
- "Incorrect type: expected Request, Response or Failure, got %s: %r"
- % (type(response), response)
- )
+ def _handle_downloader_output(
+ self, result: Union[Request, Response, Failure], request: Request
+ ) -> Optional[Deferred]:
+ assert self.spider is not None # typing
+
+ if not isinstance(result, (Request, Response, Failure)):
+ raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
+
# downloader middleware can return requests (for example, redirects)
- if isinstance(response, Request):
- self.crawl(response, spider)
- return
- # response is a Response or Failure
- d = self.scraper.enqueue_scrape(response, request, spider)
- d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
- exc_info=failure_to_exc_info(f),
- extra={'spider': spider}))
+ if isinstance(result, Request):
+ self.crawl(result)
+ return None
+
+ d = self.scraper.enqueue_scrape(result, request, self.spider)
+ d.addErrback(
+ lambda f: logger.error(
+ "Error while enqueuing downloader output",
+ exc_info=failure_to_exc_info(f),
+ extra={'spider': self.spider},
+ )
+ )
return d
- def spider_is_idle(self, spider):
- if not self.scraper.slot.is_idle():
- # scraper is not idle
+ def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
+ if spider is not None:
+ warnings.warn(
+ "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if self.slot is None:
+ raise RuntimeError("Engine slot not assigned")
+ if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
return False
-
- if self.downloader.active:
- # downloader has pending requests
+ if self.downloader.active: # downloader has pending requests
return False
-
- if self.slot.start_requests is not None:
- # not all start requests are handled
+ if self.slot.start_requests is not None: # not all start requests are handled
return False
-
if self.slot.scheduler.has_pending_requests():
- # scheduler has pending requests
return False
-
return True
- @property
- def open_spiders(self):
- return [self.spider] if self.spider else []
+ def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
+ """Inject the request into the spider <-> downloader pipeline"""
+ if spider is not None:
+ warnings.warn(
+ "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if spider is not self.spider:
+ raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
+ if self.spider is None:
+ raise RuntimeError(f"No open spider to crawl: {request}")
+ self._schedule_request(request, self.spider)
+ self.slot.nextcall.schedule() # type: ignore[union-attr]
- def has_capacity(self):
- """Does the engine have capacity to handle more spiders"""
- return not bool(self.slot)
-
- def crawl(self, request, spider):
- if spider not in self.open_spiders:
- raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
- self.schedule(request, spider)
- self.slot.nextcall.schedule()
-
- def schedule(self, request, spider):
+ def _schedule_request(self, request: Request, spider: Spider) -> None:
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
- if not self.slot.scheduler.enqueue_request(request):
+ if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
- def download(self, request, spider):
- d = self._download(request, spider)
- d.addBoth(self._downloaded, self.slot, request, spider)
- return d
+ def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
+ """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
+ if spider is None:
+ spider = self.spider
+ else:
+ warnings.warn(
+ "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if spider is not self.spider:
+ logger.warning("The spider '%s' does not match the open spider", spider.name)
+ if spider is None:
+ raise RuntimeError(f"No open spider to crawl: {request}")
+ return self._download(request, spider).addBoth(self._downloaded, request, spider)
- def _downloaded(self, response, slot, request, spider):
- slot.remove_request(request)
- return self.download(response, spider) if isinstance(response, Request) else response
+ def _downloaded(
+ self, result: Union[Response, Request], request: Request, spider: Spider
+ ) -> Union[Deferred, Response]:
+ assert self.slot is not None # typing
+ self.slot.remove_request(request)
+ return self.download(result, spider) if isinstance(result, Request) else result
- def _download(self, request, spider):
- slot = self.slot
- slot.add_request(request)
+ def _download(self, request: Request, spider: Spider) -> Deferred:
+ assert self.slot is not None # typing
- def _on_success(response):
- if not isinstance(response, (Response, Request)):
- raise TypeError(
- "Incorrect type: expected Response or Request, got %s: %r"
- % (type(response), response)
- )
- if isinstance(response, Response):
- response.request = request # tie request to response received
- logkws = self.logformatter.crawled(request, response, spider)
+ self.slot.add_request(request)
+
+ def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
+ if not isinstance(result, (Response, Request)):
+ raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
+ if isinstance(result, Response):
+ if result.request is None:
+ result.request = request
+ logkws = self.logformatter.crawled(result.request, result, spider)
if logkws is not None:
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
- self.signals.send_catch_log(signals.response_received,
- response=response, request=request, spider=spider)
- return response
+ logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
+ self.signals.send_catch_log(
+ signal=signals.response_received,
+ response=result,
+ request=result.request,
+ spider=spider,
+ )
+ return result
def _on_complete(_):
- slot.nextcall.schedule()
+ self.slot.nextcall.schedule()
return _
dwld = self.downloader.fetch(request, spider)
@@ -258,58 +302,53 @@ class ExecutionEngine:
dwld.addBoth(_on_complete)
return dwld
- @defer.inlineCallbacks
- def open_spider(self, spider, start_requests=(), close_if_idle=True):
- if not self.has_capacity():
- raise RuntimeError("No free spider slot when opening %r" % spider.name)
+ @inlineCallbacks
+ def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
+ if self.slot is not None:
+ raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider})
- nextcall = CallLaterOnce(self._next_request, spider)
- scheduler = self.scheduler_cls.from_crawler(self.crawler)
+ nextcall = CallLaterOnce(self._next_request)
+ scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
- slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
- self.slot = slot
+ self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
- yield scheduler.open(spider)
+ if hasattr(scheduler, "open"):
+ yield scheduler.open(spider)
yield self.scraper.open_spider(spider)
self.crawler.stats.open_spider(spider)
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
- slot.nextcall.schedule()
- slot.heartbeat.start(5)
+ self.slot.nextcall.schedule()
+ self.slot.heartbeat.start(5)
- def _spider_idle(self, spider):
- """Called when a spider gets idle. This function is called when there
- are no remaining pages to download or schedule. It can be called
- multiple times. If some extension raises a DontCloseSpider exception
- (in the spider_idle signal handler) the spider is not closed until the
- next loop and this function is guaranteed to be called (at least) once
- again for this spider.
+ def _spider_idle(self) -> None:
"""
- res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
+ Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
+ It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
+ exception, the spider is not closed until the next loop and this function is guaranteed to be called
+ (at least) once again.
+ """
+ assert self.spider is not None # typing
+ res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
- return
+ return None
+ if self.spider_is_idle():
+ self.close_spider(self.spider, reason='finished')
- if self.spider_is_idle(spider):
- self.close_spider(spider, reason='finished')
-
- def close_spider(self, spider, reason='cancelled'):
+ def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
"""Close (cancel) spider and clear all its outstanding requests"""
+ if self.slot is None:
+ raise RuntimeError("Engine slot not assigned")
- slot = self.slot
- if slot.closing:
- return slot.closing
- logger.info("Closing spider (%(reason)s)",
- {'reason': reason},
- extra={'spider': spider})
+ if self.slot.closing is not None:
+ return self.slot.closing
- dfd = slot.close()
+ logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
- def log_failure(msg):
- def errback(failure):
- logger.error(
- msg,
- exc_info=failure_to_exc_info(failure),
- extra={'spider': spider}
- )
+ dfd = self.slot.close()
+
+ def log_failure(msg: str) -> Callable:
+ def errback(failure: Failure) -> None:
+ logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
return errback
dfd.addBoth(lambda _: self.downloader.close())
@@ -318,19 +357,19 @@ class ExecutionEngine:
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
dfd.addErrback(log_failure('Scraper close failure'))
- dfd.addBoth(lambda _: slot.scheduler.close(reason))
- dfd.addErrback(log_failure('Scheduler close failure'))
+ if hasattr(self.slot.scheduler, "close"):
+ dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
+ dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
- signal=signals.spider_closed, spider=spider, reason=reason))
+ signal=signals.spider_closed, spider=spider, reason=reason,
+ ))
dfd.addErrback(log_failure('Error while sending spider_close signal'))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
dfd.addErrback(log_failure('Stats close failure'))
- dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
- {'reason': reason},
- extra={'spider': spider}))
+ dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
dfd.addBoth(lambda _: setattr(self, 'slot', None))
dfd.addErrback(log_failure('Error while unassigning slot'))
@@ -342,12 +381,26 @@ class ExecutionEngine:
return dfd
- def _close_all_spiders(self):
- dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
- dlist = defer.DeferredList(dfds)
- return dlist
+ @property
+ def open_spiders(self) -> list:
+ warnings.warn(
+ "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return [self.spider] if self.spider is not None else []
- @defer.inlineCallbacks
- def _finish_stopping_engine(self):
- yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
- self._closewait.callback(None)
+ def has_capacity(self) -> bool:
+ warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
+ return not bool(self.slot)
+
+ def schedule(self, request: Request, spider: Spider) -> None:
+ warnings.warn(
+ "ExecutionEngine.schedule is deprecated, please use "
+ "ExecutionEngine.crawl or ExecutionEngine.download instead",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ if self.slot is None:
+ raise RuntimeError("Engine slot not assigned")
+ self._schedule_request(request, spider)
diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
new file mode 100644
index 000000000..f7b0c3f99
--- /dev/null
+++ b/scrapy/core/http2/agent.py
@@ -0,0 +1,157 @@
+from collections import deque
+from typing import Deque, Dict, List, Optional, Tuple
+
+from twisted.internet import defer
+from twisted.internet.base import ReactorBase
+from twisted.internet.defer import Deferred
+from twisted.internet.endpoints import HostnameEndpoint
+from twisted.python.failure import Failure
+from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.error import SchemeNotSupported
+
+from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
+from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
+from scrapy.http.request import Request
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
+
+
+class H2ConnectionPool:
+ def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
+ self._reactor = reactor
+ self.settings = settings
+
+ # Store a dictionary which is used to get the respective
+ # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port)
+ self._connections: Dict[Tuple, H2ClientProtocol] = {}
+
+ # Save all requests that arrive before the connection is established
+ self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
+
+ def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
+ if key in self._pending_requests:
+ # Received a request while connecting to remote
+ # Create a deferred which will fire with the H2ClientProtocol
+ # instance
+ d = Deferred()
+ self._pending_requests[key].append(d)
+ return d
+
+ # Check if we already have a connection to the remote
+ conn = self._connections.get(key, None)
+ if conn:
+ # Return this connection instance wrapped inside a deferred
+ return defer.succeed(conn)
+
+ # No connection is established for the given URI
+ return self._new_connection(key, uri, endpoint)
+
+ def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
+ self._pending_requests[key] = deque()
+
+ conn_lost_deferred = Deferred()
+ conn_lost_deferred.addCallback(self._remove_connection, key)
+
+ factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
+ conn_d = endpoint.connect(factory)
+ conn_d.addCallback(self.put_connection, key)
+
+ d = Deferred()
+ self._pending_requests[key].append(d)
+ return d
+
+ def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
+ self._connections[key] = conn
+
+ # Now as we have established a proper HTTP/2 connection
+ # we fire all the deferred's with the connection instance
+ pending_requests = self._pending_requests.pop(key, None)
+ while pending_requests:
+ d = pending_requests.popleft()
+ d.callback(conn)
+
+ return conn
+
+ def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
+ self._connections.pop(key)
+
+ # Call the errback of all the pending requests for this connection
+ pending_requests = self._pending_requests.pop(key, None)
+ while pending_requests:
+ d = pending_requests.popleft()
+ d.errback(errors)
+
+ def close_connections(self) -> None:
+ """Close all the HTTP/2 connections and remove them from pool
+
+ Returns:
+ Deferred that fires when all connections have been closed
+ """
+ for conn in self._connections.values():
+ conn.transport.abortConnection()
+
+
+class H2Agent:
+ def __init__(
+ self,
+ reactor: ReactorBase,
+ pool: H2ConnectionPool,
+ context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
+ connect_timeout: Optional[float] = None,
+ bind_address: Optional[bytes] = None,
+ ) -> None:
+ self._reactor = reactor
+ self._pool = pool
+ self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
+ self.endpoint_factory = _StandardEndpointFactory(
+ self._reactor, self._context_factory, connect_timeout, bind_address
+ )
+
+ def get_endpoint(self, uri: URI):
+ return self.endpoint_factory.endpointForURI(uri)
+
+ def get_key(self, uri: URI) -> Tuple:
+ """
+ Arguments:
+ uri - URI obtained directly from request URL
+ """
+ return uri.scheme, uri.host, uri.port
+
+ def request(self, request: Request, spider: Spider) -> Deferred:
+ uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
+ try:
+ endpoint = self.get_endpoint(uri)
+ except SchemeNotSupported:
+ return defer.fail(Failure())
+
+ key = self.get_key(uri)
+ d = self._pool.get_connection(key, uri, endpoint)
+ d.addCallback(lambda conn: conn.request(request, spider))
+ return d
+
+
+class ScrapyProxyH2Agent(H2Agent):
+ def __init__(
+ self,
+ reactor: ReactorBase,
+ proxy_uri: URI,
+ pool: H2ConnectionPool,
+ context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
+ connect_timeout: Optional[float] = None,
+ bind_address: Optional[bytes] = None,
+ ) -> None:
+ super(ScrapyProxyH2Agent, self).__init__(
+ reactor=reactor,
+ pool=pool,
+ context_factory=context_factory,
+ connect_timeout=connect_timeout,
+ bind_address=bind_address,
+ )
+ self._proxy_uri = proxy_uri
+
+ def get_endpoint(self, uri: URI):
+ return self.endpoint_factory.endpointForURI(self._proxy_uri)
+
+ def get_key(self, uri: URI) -> Tuple:
+ """We use the proxy uri instead of uri obtained from request url"""
+ return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
new file mode 100644
index 000000000..1d150b7ce
--- /dev/null
+++ b/scrapy/core/http2/protocol.py
@@ -0,0 +1,418 @@
+import ipaddress
+import itertools
+import logging
+from collections import deque
+from ipaddress import IPv4Address, IPv6Address
+from typing import Dict, List, Optional, Union
+
+from h2.config import H2Configuration
+from h2.connection import H2Connection
+from h2.errors import ErrorCodes
+from h2.events import (
+ Event, ConnectionTerminated, DataReceived, ResponseReceived,
+ SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
+ WindowUpdated
+)
+from h2.exceptions import FrameTooLargeError, H2Error
+from twisted.internet.defer import Deferred
+from twisted.internet.error import TimeoutError
+from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
+from twisted.internet.protocol import connectionDone, Factory, Protocol
+from twisted.internet.ssl import Certificate
+from twisted.protocols.policies import TimeoutMixin
+from twisted.python.failure import Failure
+from twisted.web.client import URI
+from zope.interface import implementer
+
+from scrapy.core.http2.stream import Stream, StreamCloseReason
+from scrapy.http import Request
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
+
+
+logger = logging.getLogger(__name__)
+
+
+PROTOCOL_NAME = b"h2"
+
+
+class InvalidNegotiatedProtocol(H2Error):
+
+ def __init__(self, negotiated_protocol: bytes) -> None:
+ self.negotiated_protocol = negotiated_protocol
+
+ def __str__(self) -> str:
+ return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
+
+
+class RemoteTerminatedConnection(H2Error):
+ def __init__(
+ self,
+ remote_ip_address: Optional[Union[IPv4Address, IPv6Address]],
+ event: ConnectionTerminated,
+ ) -> None:
+ self.remote_ip_address = remote_ip_address
+ self.terminate_event = event
+
+ def __str__(self) -> str:
+ return f'Received GOAWAY frame from {self.remote_ip_address!r}'
+
+
+class MethodNotAllowed405(H2Error):
+ def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
+ self.remote_ip_address = remote_ip_address
+
+ def __str__(self) -> str:
+ return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
+
+
+@implementer(IHandshakeListener)
+class H2ClientProtocol(Protocol, TimeoutMixin):
+ IDLE_TIMEOUT = 240
+
+ def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
+ """
+ Arguments:
+ uri -- URI of the base url to which HTTP/2 Connection will be made.
+ uri is used to verify that incoming client requests have correct
+ base URL.
+ settings -- Scrapy project settings
+ conn_lost_deferred -- Deferred fires with the reason: Failure to notify
+ that connection was lost
+ """
+ self._conn_lost_deferred = conn_lost_deferred
+
+ config = H2Configuration(client_side=True, header_encoding='utf-8')
+ self.conn = H2Connection(config=config)
+
+ # ID of the next request stream
+ # Following the convention - 'Streams initiated by a client MUST
+ # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1)
+ self._stream_id_generator = itertools.count(start=1, step=2)
+
+ # Streams are stored in a dictionary keyed off their stream IDs
+ self.streams: Dict[int, Stream] = {}
+
+ # If requests are received before connection is made we keep
+ # all requests in a pool and send them as the connection is made
+ self._pending_request_stream_pool: deque = deque()
+
+ # Save an instance of errors raised which lead to losing the connection
+ # We pass these instances to the streams ResponseFailed() failure
+ self._conn_lost_errors: List[BaseException] = []
+
+ # Some meta data of this connection
+ # initialized when connection is successfully made
+ self.metadata: Dict = {
+ # Peer certificate instance
+ 'certificate': None,
+
+ # Address of the server we are connected to which
+ # is updated when HTTP/2 connection is made successfully
+ 'ip_address': None,
+
+ # URI of the peer HTTP/2 connection is made
+ 'uri': uri,
+
+ # Both ip_address and uri are used by the Stream before
+ # initiating the request to verify that the base address
+
+ # Variables taken from Project Settings
+ 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
+ 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
+
+ # Counter to keep track of opened streams. This counter
+ # is used to make sure that not more than MAX_CONCURRENT_STREAMS
+ # streams are opened which leads to ProtocolError
+ # We use simple FIFO policy to handle pending requests
+ 'active_streams': 0,
+
+ # Flag to keep track if settings were acknowledged by the remote
+ # This ensures that we have established a HTTP/2 connection
+ 'settings_acknowledged': False,
+ }
+
+ @property
+ def h2_connected(self) -> bool:
+ """Boolean to keep track of the connection status.
+ This is used while initiating pending streams to make sure
+ that we initiate stream only during active HTTP/2 Connection
+ """
+ return bool(self.transport.connected) and self.metadata['settings_acknowledged']
+
+ @property
+ def allowed_max_concurrent_streams(self) -> int:
+ """We keep total two streams for client (sending data) and
+ server side (receiving data) for a single request. To be safe
+ we choose the minimum. Since this value can change in event
+ RemoteSettingsChanged we make variable a property.
+ """
+ return min(
+ self.conn.local_settings.max_concurrent_streams,
+ self.conn.remote_settings.max_concurrent_streams
+ )
+
+ def _send_pending_requests(self) -> None:
+ """Initiate all pending requests from the deque following FIFO
+ We make sure that at any time {allowed_max_concurrent_streams}
+ streams are active.
+ """
+ while (
+ self._pending_request_stream_pool
+ and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
+ and self.h2_connected
+ ):
+ self.metadata['active_streams'] += 1
+ stream = self._pending_request_stream_pool.popleft()
+ stream.initiate_request()
+ self._write_to_transport()
+
+ def pop_stream(self, stream_id: int) -> Stream:
+ """Perform cleanup when a stream is closed
+ """
+ stream = self.streams.pop(stream_id)
+ self.metadata['active_streams'] -= 1
+ self._send_pending_requests()
+ return stream
+
+ def _new_stream(self, request: Request, spider: Spider) -> Stream:
+ """Instantiates a new Stream object
+ """
+ stream = Stream(
+ stream_id=next(self._stream_id_generator),
+ request=request,
+ protocol=self,
+ download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
+ download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
+ )
+ self.streams[stream.stream_id] = stream
+ return stream
+
+ def _write_to_transport(self) -> None:
+ """ Write data to the underlying transport connection
+ from the HTTP2 connection instance if any
+ """
+ # Reset the idle timeout as connection is still actively sending data
+ self.resetTimeout()
+
+ data = self.conn.data_to_send()
+ self.transport.write(data)
+
+ def request(self, request: Request, spider: Spider) -> Deferred:
+ if not isinstance(request, Request):
+ raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
+
+ stream = self._new_stream(request, spider)
+ d = stream.get_response()
+
+ # Add the stream to the request pool
+ self._pending_request_stream_pool.append(stream)
+
+ # If we receive a request when connection is idle
+ # We need to initiate pending requests
+ self._send_pending_requests()
+ return d
+
+ def connectionMade(self) -> None:
+ """Called by Twisted when the connection is established. We can start
+ sending some data now: we should open with the connection preamble.
+ """
+ # Initialize the timeout
+ self.setTimeout(self.IDLE_TIMEOUT)
+
+ destination = self.transport.getPeer()
+ self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
+
+ # Initiate H2 Connection
+ self.conn.initiate_connection()
+ self._write_to_transport()
+
+ def _lose_connection_with_error(self, errors: List[BaseException]) -> None:
+ """Helper function to lose the connection with the error sent as a
+ reason"""
+ self._conn_lost_errors += errors
+ self.transport.loseConnection()
+
+ def handshakeCompleted(self) -> None:
+ """
+ Close the connection if it's not made via the expected protocol
+ """
+ if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
+ # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
+ self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
+
+ def _check_received_data(self, data: bytes) -> None:
+ """Checks for edge cases where the connection to remote fails
+ without raising an appropriate H2Error
+
+ Arguments:
+ data -- Data received from the remote
+ """
+ if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
+ raise MethodNotAllowed405(self.metadata['ip_address'])
+
+ def dataReceived(self, data: bytes) -> None:
+ # Reset the idle timeout as connection is still actively receiving data
+ self.resetTimeout()
+
+ try:
+ self._check_received_data(data)
+ events = self.conn.receive_data(data)
+ self._handle_events(events)
+ except H2Error as e:
+ if isinstance(e, FrameTooLargeError):
+ # hyper-h2 does not drop the connection in this scenario, we
+ # need to abort the connection manually.
+ self._conn_lost_errors += [e]
+ self.transport.abortConnection()
+ return
+
+ # Save this error as ultimately the connection will be dropped
+ # internally by hyper-h2. Saved error will be passed to all the streams
+ # closed with the connection.
+ self._lose_connection_with_error([e])
+ finally:
+ self._write_to_transport()
+
+ def timeoutConnection(self) -> None:
+ """Called when the connection times out.
+ We lose the connection with TimeoutError"""
+
+ # Check whether there are open streams. If there are, we're going to
+ # want to use the error code PROTOCOL_ERROR. If there aren't, use
+ # NO_ERROR.
+ if (
+ self.conn.open_outbound_streams > 0
+ or self.conn.open_inbound_streams > 0
+ or self.metadata['active_streams'] > 0
+ ):
+ error_code = ErrorCodes.PROTOCOL_ERROR
+ else:
+ error_code = ErrorCodes.NO_ERROR
+ self.conn.close_connection(error_code=error_code)
+ self._write_to_transport()
+
+ self._lose_connection_with_error([
+ TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
+ ])
+
+ def connectionLost(self, reason: Failure = connectionDone) -> None:
+ """Called by Twisted when the transport connection is lost.
+ No need to write anything to transport here.
+ """
+ # Cancel the timeout if not done yet
+ self.setTimeout(None)
+
+ # Notify the connection pool instance such that no new requests are
+ # sent over current connection
+ if not reason.check(connectionDone):
+ self._conn_lost_errors.append(reason)
+
+ self._conn_lost_deferred.callback(self._conn_lost_errors)
+
+ for stream in self.streams.values():
+ if stream.metadata['request_sent']:
+ close_reason = StreamCloseReason.CONNECTION_LOST
+ else:
+ close_reason = StreamCloseReason.INACTIVE
+ stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
+
+ self.metadata['active_streams'] -= len(self.streams)
+ self.streams.clear()
+ self._pending_request_stream_pool.clear()
+ self.conn.close_connection()
+
+ def _handle_events(self, events: List[Event]) -> None:
+ """Private method which acts as a bridge between the events
+ received from the HTTP/2 data and IH2EventsHandler
+
+ Arguments:
+ events -- A list of events that the remote peer triggered by sending data
+ """
+ for event in events:
+ if isinstance(event, ConnectionTerminated):
+ self.connection_terminated(event)
+ elif isinstance(event, DataReceived):
+ self.data_received(event)
+ elif isinstance(event, ResponseReceived):
+ self.response_received(event)
+ elif isinstance(event, StreamEnded):
+ self.stream_ended(event)
+ elif isinstance(event, StreamReset):
+ self.stream_reset(event)
+ elif isinstance(event, WindowUpdated):
+ self.window_updated(event)
+ elif isinstance(event, SettingsAcknowledged):
+ self.settings_acknowledged(event)
+ elif isinstance(event, UnknownFrameReceived):
+ logger.warning('Unknown frame received: %s', event.frame)
+
+ # Event handler functions starts here
+ def connection_terminated(self, event: ConnectionTerminated) -> None:
+ self._lose_connection_with_error([
+ RemoteTerminatedConnection(self.metadata['ip_address'], event)
+ ])
+
+ def data_received(self, event: DataReceived) -> None:
+ try:
+ stream = self.streams[event.stream_id]
+ except KeyError:
+ pass # We ignore server-initiated events
+ else:
+ stream.receive_data(event.data, event.flow_controlled_length)
+
+ def response_received(self, event: ResponseReceived) -> None:
+ try:
+ stream = self.streams[event.stream_id]
+ except KeyError:
+ pass # We ignore server-initiated events
+ else:
+ stream.receive_headers(event.headers)
+
+ def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
+ self.metadata['settings_acknowledged'] = True
+
+ # Send off all the pending requests as now we have
+ # established a proper HTTP/2 connection
+ self._send_pending_requests()
+
+ # Update certificate when our HTTP/2 connection is established
+ self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
+
+ def stream_ended(self, event: StreamEnded) -> None:
+ try:
+ stream = self.pop_stream(event.stream_id)
+ except KeyError:
+ pass # We ignore server-initiated events
+ else:
+ stream.close(StreamCloseReason.ENDED, from_protocol=True)
+
+ def stream_reset(self, event: StreamReset) -> None:
+ try:
+ stream = self.pop_stream(event.stream_id)
+ except KeyError:
+ pass # We ignore server-initiated events
+ else:
+ stream.close(StreamCloseReason.RESET, from_protocol=True)
+
+ def window_updated(self, event: WindowUpdated) -> None:
+ if event.stream_id != 0:
+ self.streams[event.stream_id].receive_window_update()
+ else:
+ # Send leftover data for all the streams
+ for stream in self.streams.values():
+ stream.receive_window_update()
+
+
+@implementer(IProtocolNegotiationFactory)
+class H2ClientFactory(Factory):
+ def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
+ self.uri = uri
+ self.settings = settings
+ self.conn_lost_deferred = conn_lost_deferred
+
+ def buildProtocol(self, addr) -> H2ClientProtocol:
+ return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
+
+ def acceptableProtocols(self) -> List[bytes]:
+ return [PROTOCOL_NAME]
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
new file mode 100644
index 000000000..c2a4b702f
--- /dev/null
+++ b/scrapy/core/http2/stream.py
@@ -0,0 +1,470 @@
+import logging
+from enum import Enum
+from io import BytesIO
+from urllib.parse import urlparse
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
+
+from h2.errors import ErrorCodes
+from h2.exceptions import H2Error, ProtocolError, StreamClosedError
+from hpack import HeaderTuple
+from twisted.internet.defer import Deferred, CancelledError
+from twisted.internet.error import ConnectionClosed
+from twisted.python.failure import Failure
+from twisted.web.client import ResponseFailed
+
+from scrapy.http import Request
+from scrapy.http.headers import Headers
+from scrapy.responsetypes import responsetypes
+
+if TYPE_CHECKING:
+ from scrapy.core.http2.protocol import H2ClientProtocol
+
+
+logger = logging.getLogger(__name__)
+
+
+class InactiveStreamClosed(ConnectionClosed):
+ """Connection was closed without sending request headers
+ of the stream. This happens when a stream is waiting for other
+ streams to close and connection is lost."""
+
+ def __init__(self, request: Request) -> None:
+ self.request = request
+
+ def __str__(self) -> str:
+ return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
+
+
+class InvalidHostname(H2Error):
+
+ def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
+ self.request = request
+ self.expected_hostname = expected_hostname
+ self.expected_netloc = expected_netloc
+
+ def __str__(self) -> str:
+ return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
+
+
+class StreamCloseReason(Enum):
+ # Received a StreamEnded event from the remote
+ ENDED = 1
+
+ # Received a StreamReset event -- ended abruptly
+ RESET = 2
+
+ # Transport connection was lost
+ CONNECTION_LOST = 3
+
+ # Expected response body size is more than allowed limit
+ MAXSIZE_EXCEEDED = 4
+
+ # Response deferred is cancelled by the client
+ # (happens when client called response_deferred.cancel())
+ CANCELLED = 5
+
+ # Connection lost and the stream was not initiated
+ INACTIVE = 6
+
+ # The hostname of the request is not same as of connected peer hostname
+ # As a result sending this request will the end the connection
+ INVALID_HOSTNAME = 7
+
+
+class Stream:
+ """Represents a single HTTP/2 Stream.
+
+ Stream is a bidirectional flow of bytes within an established connection,
+ which may carry one or more messages. Handles the transfer of HTTP Headers
+ and Data frames.
+
+ Role of this class is to
+ 1. Combine all the data frames
+ """
+
+ def __init__(
+ self,
+ stream_id: int,
+ request: Request,
+ protocol: "H2ClientProtocol",
+ download_maxsize: int = 0,
+ download_warnsize: int = 0,
+ ) -> None:
+ """
+ Arguments:
+ stream_id -- Unique identifier for the stream within a single HTTP/2 connection
+ request -- The HTTP request associated to the stream
+ protocol -- Parent H2ClientProtocol instance
+ """
+ self.stream_id: int = stream_id
+ self._request: Request = request
+ self._protocol: "H2ClientProtocol" = protocol
+
+ self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
+ self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
+
+ # Metadata of an HTTP/2 connection stream
+ # initialized when stream is instantiated
+ self.metadata: Dict = {
+ 'request_content_length': 0 if self._request.body is None else len(self._request.body),
+
+ # Flag to keep track whether the stream has initiated the request
+ 'request_sent': False,
+
+ # Flag to track whether we have logged about exceeding download warnsize
+ 'reached_warnsize': False,
+
+ # Each time we send a data frame, we will decrease value by the amount send.
+ 'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
+
+ # Flag to keep track whether client (self) have closed this stream
+ 'stream_closed_local': False,
+
+ # Flag to keep track whether the server has closed the stream
+ 'stream_closed_server': False,
+ }
+
+ # Private variable used to build the response
+ # this response is then converted to appropriate Response class
+ # passed to the response deferred callback
+ self._response: Dict = {
+ # Data received frame by frame from the server is appended
+ # and passed to the response Deferred when completely received.
+ 'body': BytesIO(),
+
+ # The amount of data received that counts against the
+ # flow control window
+ 'flow_controlled_size': 0,
+
+ # Headers received after sending the request
+ 'headers': Headers({}),
+ }
+
+ def _cancel(_) -> None:
+ # Close this stream as gracefully as possible
+ # If the associated request is initiated we reset this stream
+ # else we directly call close() method
+ if self.metadata['request_sent']:
+ self.reset_stream(StreamCloseReason.CANCELLED)
+ else:
+ self.close(StreamCloseReason.CANCELLED)
+
+ self._deferred_response = Deferred(_cancel)
+
+ def __str__(self) -> str:
+ return f'Stream(id={self.stream_id!r})'
+
+ __repr__ = __str__
+
+ @property
+ def _log_warnsize(self) -> bool:
+ """Checks if we have received data which exceeds the download warnsize
+ and whether we have not already logged about it.
+
+ Returns:
+ True if both the above conditions hold true
+ False if any of the conditions is false
+ """
+ content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
+ return (
+ self._download_warnsize
+ and (
+ self._response['flow_controlled_size'] > self._download_warnsize
+ or content_length_header > self._download_warnsize
+ )
+ and not self.metadata['reached_warnsize']
+ )
+
+ def get_response(self) -> Deferred:
+ """Simply return a Deferred which fires when response
+ from the asynchronous request is available
+ """
+ return self._deferred_response
+
+ def check_request_url(self) -> bool:
+ # Make sure that we are sending the request to the correct URL
+ url = urlparse(self._request.url)
+ return (
+ url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
+ or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
+ or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
+ )
+
+ def _get_request_headers(self) -> List[Tuple[str, str]]:
+ url = urlparse(self._request.url)
+
+ path = url.path
+ if url.query:
+ path += '?' + url.query
+
+ # This pseudo-header field MUST NOT be empty for "http" or "https"
+ # URIs; "http" or "https" URIs that do not contain a path component
+ # MUST include a value of '/'. The exception to this rule is an
+ # OPTIONS request for an "http" or "https" URI that does not include
+ # a path component; these MUST include a ":path" pseudo-header field
+ # with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
+ if not path:
+ path = '*' if self._request.method == 'OPTIONS' else '/'
+
+ # Make sure pseudo-headers comes before all the other headers
+ headers = [
+ (':method', self._request.method),
+ (':authority', url.netloc),
+ ]
+
+ # The ":scheme" and ":path" pseudo-header fields MUST
+ # be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
+ if self._request.method != 'CONNECT':
+ headers += [
+ (':scheme', self._protocol.metadata['uri'].scheme),
+ (':path', path),
+ ]
+
+ content_length = str(len(self._request.body))
+ headers.append(('Content-Length', content_length))
+
+ content_length_name = self._request.headers.normkey(b'Content-Length')
+ for name, values in self._request.headers.items():
+ for value in values:
+ value = str(value, 'utf-8')
+ if name == content_length_name:
+ if value != content_length:
+ logger.warning(
+ 'Ignoring bad Content-Length header %r of request %r, '
+ 'sending %r instead',
+ value,
+ self._request,
+ content_length,
+ )
+ continue
+ headers.append((str(name, 'utf-8'), value))
+
+ return headers
+
+ def initiate_request(self) -> None:
+ if self.check_request_url():
+ headers = self._get_request_headers()
+ self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
+ self.metadata['request_sent'] = True
+ self.send_data()
+ else:
+ # Close this stream calling the response errback
+ # Note that we have not sent any headers
+ self.close(StreamCloseReason.INVALID_HOSTNAME)
+
+ def send_data(self) -> None:
+ """Called immediately after the headers are sent. Here we send all the
+ data as part of the request.
+
+ If the content length is 0 initially then we end the stream immediately and
+ wait for response data.
+
+ Warning: Only call this method when stream not closed from client side
+ and has initiated request already by sending HEADER frame. If not then
+ stream will raise ProtocolError (raise by h2 state machine).
+ """
+ if self.metadata['stream_closed_local']:
+ raise StreamClosedError(self.stream_id)
+
+ # Firstly, check what the flow control window is for current stream.
+ window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
+
+ # Next, check what the maximum frame size is.
+ max_frame_size = self._protocol.conn.max_outbound_frame_size
+
+ # We will send no more than the window size or the remaining file size
+ # of data in this call, whichever is smaller.
+ bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
+
+ # We now need to send a number of data frames.
+ while bytes_to_send_size > 0:
+ chunk_size = min(bytes_to_send_size, max_frame_size)
+
+ data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
+ data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
+
+ self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
+
+ bytes_to_send_size = bytes_to_send_size - chunk_size
+ self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
+
+ self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
+
+ # End the stream if no more data needs to be send
+ if self.metadata['remaining_content_length'] == 0:
+ self._protocol.conn.end_stream(self.stream_id)
+
+ # Q. What about the rest of the data?
+ # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
+
+ def receive_window_update(self) -> None:
+ """Flow control window size was changed.
+ Send data that earlier could not be sent as we were
+ blocked behind the flow control.
+ """
+ if (
+ self.metadata['remaining_content_length']
+ and not self.metadata['stream_closed_server']
+ and self.metadata['request_sent']
+ ):
+ self.send_data()
+
+ def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
+ self._response['body'].write(data)
+ self._response['flow_controlled_size'] += flow_controlled_length
+
+ # We check maxsize here in case the Content-Length header was not received
+ if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
+ self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
+ return
+
+ if self._log_warnsize:
+ self.metadata['reached_warnsize'] = True
+ warning_msg = (
+ f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
+ f'warn size ({self._download_warnsize}) in request {self._request}'
+ )
+ logger.warning(warning_msg)
+
+ # Acknowledge the data received
+ self._protocol.conn.acknowledge_received_data(
+ self._response['flow_controlled_size'],
+ self.stream_id
+ )
+
+ def receive_headers(self, headers: List[HeaderTuple]) -> None:
+ for name, value in headers:
+ self._response['headers'][name] = value
+
+ # Check if we exceed the allowed max data size which can be received
+ expected_size = int(self._response['headers'].get(b'Content-Length', -1))
+ if self._download_maxsize and expected_size > self._download_maxsize:
+ self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
+ return
+
+ if self._log_warnsize:
+ self.metadata['reached_warnsize'] = True
+ warning_msg = (
+ f'Expected response size ({expected_size}) larger than '
+ f'download warn size ({self._download_warnsize}) in request {self._request}'
+ )
+ logger.warning(warning_msg)
+
+ def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
+ """Close this stream by sending a RST_FRAME to the remote peer"""
+ if self.metadata['stream_closed_local']:
+ raise StreamClosedError(self.stream_id)
+
+ # Clear buffer earlier to avoid keeping data in memory for a long time
+ self._response['body'].truncate(0)
+
+ self.metadata['stream_closed_local'] = True
+ self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
+ self.close(reason)
+
+ def close(
+ self,
+ reason: StreamCloseReason,
+ errors: Optional[List[BaseException]] = None,
+ from_protocol: bool = False,
+ ) -> None:
+ """Based on the reason sent we will handle each case.
+ """
+ if self.metadata['stream_closed_server']:
+ raise StreamClosedError(self.stream_id)
+
+ if not isinstance(reason, StreamCloseReason):
+ raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
+
+ # Have default value of errors as an empty list as
+ # some cases can add a list of exceptions
+ errors = errors or []
+
+ if not from_protocol:
+ self._protocol.pop_stream(self.stream_id)
+
+ self.metadata['stream_closed_server'] = True
+
+ # We do not check for Content-Length or Transfer-Encoding in response headers
+ # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
+ # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6)
+
+ # NOTE: Order of handling the events is important here
+ # As we immediately cancel the request when maxsize is exceeded while
+ # receiving DATA_FRAME's when we have received the headers (not
+ # having Content-Length)
+ if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
+ expected_size = int(self._response['headers'].get(
+ b'Content-Length',
+ self._response['flow_controlled_size'])
+ )
+ error_msg = (
+ f'Cancelling download of {self._request.url}: received response '
+ f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
+ )
+ logger.error(error_msg)
+ self._deferred_response.errback(CancelledError(error_msg))
+
+ elif reason is StreamCloseReason.ENDED:
+ self._fire_response_deferred()
+
+ # Stream was abruptly ended here
+ elif reason is StreamCloseReason.CANCELLED:
+ # Client has cancelled the request. Remove all the data
+ # received and fire the response deferred with no flags set
+
+ # NOTE: The data is already flushed in Stream.reset_stream() called
+ # immediately when the stream needs to be cancelled
+
+ # There maybe no :status in headers, we make
+ # HTTP Status Code: 499 - Client Closed Request
+ self._response['headers'][':status'] = '499'
+ self._fire_response_deferred()
+
+ elif reason is StreamCloseReason.RESET:
+ self._deferred_response.errback(ResponseFailed([
+ Failure(
+ f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
+ ProtocolError
+ )
+ ]))
+
+ elif reason is StreamCloseReason.CONNECTION_LOST:
+ self._deferred_response.errback(ResponseFailed(errors))
+
+ elif reason is StreamCloseReason.INACTIVE:
+ errors.insert(0, InactiveStreamClosed(self._request))
+ self._deferred_response.errback(ResponseFailed(errors))
+
+ else:
+ assert reason is StreamCloseReason.INVALID_HOSTNAME
+ self._deferred_response.errback(InvalidHostname(
+ self._request,
+ str(self._protocol.metadata['uri'].host, 'utf-8'),
+ f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
+ ))
+
+ def _fire_response_deferred(self) -> None:
+ """Builds response from the self._response dict
+ and fires the response deferred callback with the
+ generated response instance"""
+
+ body = self._response['body'].getvalue()
+ response_cls = responsetypes.from_args(
+ headers=self._response['headers'],
+ url=self._request.url,
+ body=body,
+ )
+
+ response = response_cls(
+ url=self._request.url,
+ status=int(self._response['headers'][':status']),
+ headers=self._response['headers'],
+ body=body,
+ request=self._request,
+ certificate=self._protocol.metadata['certificate'],
+ ip_address=self._protocol.metadata['ip_address'],
+ protocol='h2',
+ )
+
+ self._deferred_response.callback(response)
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a18c26b17..5ba0fb63b 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,46 +1,179 @@
-import os
import json
import logging
-import warnings
-from os.path import join, exists
+import os
+from abc import abstractmethod
+from os.path import exists, join
+from typing import Optional, Type, TypeVar
-from queuelib import PriorityQueue
+from twisted.internet.defer import Deferred
-from scrapy.utils.misc import load_object, create_instance
+from scrapy.crawler import Crawler
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.misc import create_instance, load_object
logger = logging.getLogger(__name__)
-class Scheduler:
+class BaseSchedulerMeta(type):
"""
- Scrapy Scheduler. It allows to enqueue requests and then get
- a next request to download. Scheduler is also handling duplication
- filtering, via dupefilter.
-
- Prioritization and queueing is not performed by the Scheduler.
- User sets ``priority`` field for each Request, and a PriorityQueue
- (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
- to dequeue requests in a desired order.
-
- Scheduler uses two PriorityQueue instances, configured to work in-memory
- and on-disk (optional). When on-disk queue is present, it is used by
- default, and an in-memory queue is used as a fallback for cases where
- a disk queue can't handle a request (can't serialize it).
-
- :setting:`SCHEDULER_MEMORY_QUEUE` and
- :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
- which PriorityQueue instances would be instantiated with, to keep requests
- on disk and in memory respectively.
-
- Overall, Scheduler is an object which holds several PriorityQueue instances
- (in-memory and on-disk) and implements fallback logic for them.
- Also, it handles dupefilters.
+ Metaclass to check scheduler classes against the necessary interface
"""
- def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
- logunser=False, stats=None, pqclass=None, crawler=None):
+ def __instancecheck__(cls, instance):
+ return cls.__subclasscheck__(type(instance))
+
+ def __subclasscheck__(cls, subclass):
+ return (
+ hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
+ and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
+ and hasattr(subclass, "next_request") and callable(subclass.next_request)
+ )
+
+
+class BaseScheduler(metaclass=BaseSchedulerMeta):
+ """
+ The scheduler component is responsible for storing requests received from
+ the engine, and feeding them back upon request (also to the engine).
+
+ The original sources of said requests are:
+
+ * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks
+ * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods
+ * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods
+
+ The order in which the scheduler returns its stored requests (via the ``next_request`` method)
+ plays a great part in determining the order in which those requests are downloaded.
+
+ The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler):
+ """
+ Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
+ """
+ return cls()
+
+ def open(self, spider: Spider) -> Optional[Deferred]:
+ """
+ Called when the spider is opened by the engine. It receives the spider
+ instance as argument and it's useful to execute initialization code.
+
+ :param spider: the spider object for the current crawl
+ :type spider: :class:`~scrapy.spiders.Spider`
+ """
+ pass
+
+ def close(self, reason: str) -> Optional[Deferred]:
+ """
+ Called when the spider is closed by the engine. It receives the reason why the crawl
+ finished as argument and it's useful to execute cleaning code.
+
+ :param reason: a string which describes the reason why the spider was closed
+ :type reason: :class:`str`
+ """
+ pass
+
+ @abstractmethod
+ def has_pending_requests(self) -> bool:
+ """
+ ``True`` if the scheduler has enqueued requests, ``False`` otherwise
+ """
+ raise NotImplementedError()
+
+ @abstractmethod
+ def enqueue_request(self, request: Request) -> bool:
+ """
+ Process a request received by the engine.
+
+ Return ``True`` if the request is stored correctly, ``False`` otherwise.
+
+ If ``False``, the engine will fire a ``request_dropped`` signal, and
+ will not make further attempts to schedule the request at a later time.
+ For reference, the default Scrapy scheduler returns ``False`` when the
+ request is rejected by the dupefilter.
+ """
+ raise NotImplementedError()
+
+ @abstractmethod
+ def next_request(self) -> Optional[Request]:
+ """
+ Return the next :class:`~scrapy.http.Request` to be processed, or ``None``
+ to indicate that there are no requests to be considered ready at the moment.
+
+ Returning ``None`` implies that no request from the scheduler will be sent
+ to the downloader in the current reactor cycle. The engine will continue
+ calling ``next_request`` until ``has_pending_requests`` is ``False``.
+ """
+ raise NotImplementedError()
+
+
+SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
+
+
+class Scheduler(BaseScheduler):
+ """
+ Default Scrapy scheduler. This implementation also handles duplication
+ filtering via the :setting:`dupefilter `.
+
+ This scheduler stores requests into several priority queues (defined by the
+ :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues
+ are backed by either memory or disk based queues (respectively defined by the
+ :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings).
+
+ Request prioritization is almost entirely delegated to the priority queue. The only
+ prioritization performed by this scheduler is using the disk-based queue if present
+ (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based
+ queue if a serialization error occurs. If the disk queue is not present, the memory one
+ is used directly.
+
+ :param dupefilter: An object responsible for checking and filtering duplicate requests.
+ The value for the :setting:`DUPEFILTER_CLASS` setting is used by default.
+ :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar:
+ any class that implements the `BaseDupeFilter` interface
+
+ :param jobdir: The path of a directory to be used for persisting the crawl's state.
+ The value for the :setting:`JOBDIR` setting is used by default.
+ See :ref:`topics-jobs`.
+ :type jobdir: :class:`str` or ``None``
+
+ :param dqclass: A class to be used as persistent request queue.
+ The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
+ :type dqclass: class
+
+ :param mqclass: A class to be used as non-persistent request queue.
+ The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
+ :type mqclass: class
+
+ :param logunser: A boolean that indicates whether or not unserializable requests should be logged.
+ The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
+ :type logunser: bool
+
+ :param stats: A stats collector object to record stats about the request scheduling process.
+ The value for the :setting:`STATS_CLASS` setting is used by default.
+ :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar:
+ any class that implements the `StatsCollector` interface
+
+ :param pqclass: A class to be used as priority queue for requests.
+ The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
+ :type pqclass: class
+
+ :param crawler: The crawler object corresponding to the current crawl.
+ :type crawler: :class:`scrapy.crawler.Crawler`
+ """
+ def __init__(
+ self,
+ dupefilter,
+ jobdir: Optional[str] = None,
+ dqclass=None,
+ mqclass=None,
+ logunser: bool = False,
+ stats=None,
+ pqclass=None,
+ crawler: Optional[Crawler] = None,
+ ):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.pqclass = pqclass
@@ -51,42 +184,57 @@ class Scheduler:
self.crawler = crawler
@classmethod
- def from_crawler(cls, crawler):
- settings = crawler.settings
- dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
- dupefilter = create_instance(dupefilter_cls, settings, crawler)
- pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
- if pqclass is PriorityQueue:
- warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
- " is no longer supported because of API changes; "
- "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
- ScrapyDeprecationWarning)
- from scrapy.pqueues import ScrapyPriorityQueue
- pqclass = ScrapyPriorityQueue
+ def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
+ """
+ Factory method, initializes the scheduler with arguments taken from the crawl settings
+ """
+ dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
+ return cls(
+ dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
+ jobdir=job_dir(crawler.settings),
+ dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
+ mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
+ logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
+ stats=crawler.stats,
+ pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
+ crawler=crawler,
+ )
- dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
- mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
- logunser = settings.getbool('SCHEDULER_DEBUG')
- return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
- stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
- mqclass=mqclass, crawler=crawler)
-
- def has_pending_requests(self):
+ def has_pending_requests(self) -> bool:
return len(self) > 0
- def open(self, spider):
+ def open(self, spider: Spider) -> Optional[Deferred]:
+ """
+ (1) initialize the memory queue
+ (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
+ (3) return the result of the dupefilter's ``open`` method
+ """
self.spider = spider
self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
return self.df.open()
- def close(self, reason):
- if self.dqs:
+ def close(self, reason: str) -> Optional[Deferred]:
+ """
+ (1) dump pending requests to disk if there is a disk queue
+ (2) return the result of the dupefilter's ``close`` method
+ """
+ if self.dqs is not None:
state = self.dqs.close()
+ assert isinstance(self.dqdir, str)
self._write_dqs_state(self.dqdir, state)
return self.df.close(reason)
- def enqueue_request(self, request):
+ def enqueue_request(self, request: Request) -> bool:
+ """
+ Unless the received request is filtered out by the Dupefilter, attempt to push
+ it into the disk queue, falling back to pushing it into the memory queue.
+
+ Increment the appropriate stats, such as: ``scheduler/enqueued``,
+ ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
+
+ Return ``True`` if the request was stored successfully, ``False`` otherwise.
+ """
if not request.dont_filter and self.df.request_seen(request):
self.df.log(request, self.spider)
return False
@@ -99,24 +247,35 @@ class Scheduler:
self.stats.inc_value('scheduler/enqueued', spider=self.spider)
return True
- def next_request(self):
+ def next_request(self) -> Optional[Request]:
+ """
+ Return a :class:`~scrapy.http.Request` object from the memory queue,
+ falling back to the disk queue if the memory queue is empty.
+ Return ``None`` if there are no more enqueued requests.
+
+ Increment the appropriate stats, such as: ``scheduler/dequeued``,
+ ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
+ """
request = self.mqs.pop()
- if request:
+ if request is not None:
self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
else:
request = self._dqpop()
- if request:
+ if request is not None:
self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
- if request:
+ if request is not None:
self.stats.inc_value('scheduler/dequeued', spider=self.spider)
return request
- def __len__(self):
- return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs)
+ def __len__(self) -> int:
+ """
+ Return the total amount of enqueued requests
+ """
+ return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs)
- def _dqpush(self, request):
+ def _dqpush(self, request: Request) -> bool:
if self.dqs is None:
- return
+ return False
try:
self.dqs.push(request)
except ValueError as e: # non serializable request
@@ -127,18 +286,18 @@ class Scheduler:
logger.warning(msg, {'request': request, 'reason': e},
exc_info=True, extra={'spider': self.spider})
self.logunser = False
- self.stats.inc_value('scheduler/unserializable',
- spider=self.spider)
- return
+ self.stats.inc_value('scheduler/unserializable', spider=self.spider)
+ return False
else:
return True
- def _mqpush(self, request):
+ def _mqpush(self, request: Request) -> None:
self.mqs.push(request)
- def _dqpop(self):
- if self.dqs:
+ def _dqpop(self) -> Optional[Request]:
+ if self.dqs is not None:
return self.dqs.pop()
+ return None
def _mq(self):
""" Create a new priority queue instance, with in-memory storage """
@@ -162,21 +321,22 @@ class Scheduler:
{'queuesize': len(q)}, extra={'spider': self.spider})
return q
- def _dqdir(self, jobdir):
+ def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
- if jobdir:
+ if jobdir is not None:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
+ return None
- def _read_dqs_state(self, dqdir):
+ def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json')
if not exists(path):
- return ()
+ return []
with open(path) as f:
return json.load(f)
- def _write_dqs_state(self, dqdir, state):
+ def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f:
json.dump(state, f)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index d07c7aa62..d6d6f64f9 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -3,21 +3,25 @@ extracts information from them"""
import logging
from collections import deque
+from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
from itemadapter import is_item
-from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
-from scrapy import signals
+from scrapy import signals, Spider
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
+QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
+
+
logger = logging.getLogger(__name__)
@@ -26,46 +30,46 @@ class Slot:
MIN_RESPONSE_SIZE = 1024
- def __init__(self, max_active_size=5000000):
+ def __init__(self, max_active_size: int = 5000000):
self.max_active_size = max_active_size
- self.queue = deque()
- self.active = set()
- self.active_size = 0
- self.itemproc_size = 0
- self.closing = None
+ self.queue: Deque[QueueTuple] = deque()
+ self.active: Set[Request] = set()
+ self.active_size: int = 0
+ self.itemproc_size: int = 0
+ self.closing: Optional[Deferred] = None
- def add_response_request(self, response, request):
- deferred = defer.Deferred()
- self.queue.append((response, request, deferred))
- if isinstance(response, Response):
- self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE)
+ def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
+ deferred = Deferred()
+ self.queue.append((result, request, deferred))
+ if isinstance(result, Response):
+ self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
else:
self.active_size += self.MIN_RESPONSE_SIZE
return deferred
- def next_response_request_deferred(self):
+ def next_response_request_deferred(self) -> QueueTuple:
response, request, deferred = self.queue.popleft()
self.active.add(request)
return response, request, deferred
- def finish_response(self, response, request):
+ def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
self.active.remove(request)
- if isinstance(response, Response):
- self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE)
+ if isinstance(result, Response):
+ self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
else:
self.active_size -= self.MIN_RESPONSE_SIZE
- def is_idle(self):
+ def is_idle(self) -> bool:
return not (self.queue or self.active)
- def needs_backout(self):
+ def needs_backout(self) -> bool:
return self.active_size > self.max_active_size
class Scraper:
def __init__(self, crawler):
- self.slot = None
+ self.slot: Optional[Slot] = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
self.itemproc = itemproc_cls.from_crawler(crawler)
@@ -74,36 +78,39 @@ class Scraper:
self.signals = crawler.signals
self.logformatter = crawler.logformatter
- @defer.inlineCallbacks
- def open_spider(self, spider):
+ @inlineCallbacks
+ def open_spider(self, spider: Spider):
"""Open the given spider for scraping and allocate resources for it"""
self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
yield self.itemproc.open_spider(spider)
- def close_spider(self, spider):
+ def close_spider(self, spider: Spider) -> Deferred:
"""Close a spider being scraped and release its resources"""
- slot = self.slot
- slot.closing = defer.Deferred()
- slot.closing.addCallback(self.itemproc.close_spider)
- self._check_if_closing(spider, slot)
- return slot.closing
+ if self.slot is None:
+ raise RuntimeError("Scraper slot not assigned")
+ self.slot.closing = Deferred()
+ self.slot.closing.addCallback(self.itemproc.close_spider)
+ self._check_if_closing(spider)
+ return self.slot.closing
- def is_idle(self):
+ def is_idle(self) -> bool:
"""Return True if there isn't any more spiders to process"""
return not self.slot
- def _check_if_closing(self, spider, slot):
- if slot.closing and slot.is_idle():
- slot.closing.callback(spider)
+ def _check_if_closing(self, spider: Spider) -> None:
+ assert self.slot is not None # typing
+ if self.slot.closing and self.slot.is_idle():
+ self.slot.closing.callback(spider)
- def enqueue_scrape(self, response, request, spider):
- slot = self.slot
- dfd = slot.add_response_request(response, request)
+ def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+ if self.slot is None:
+ raise RuntimeError("Scraper slot not assigned")
+ dfd = self.slot.add_response_request(result, request)
def finish_scraping(_):
- slot.finish_response(response, request)
- self._check_if_closing(spider, slot)
- self._scrape_next(spider, slot)
+ self.slot.finish_response(result, request)
+ self._check_if_closing(spider)
+ self._scrape_next(spider)
return _
dfd.addBoth(finish_scraping)
@@ -112,51 +119,52 @@ class Scraper:
{'request': request},
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
- self._scrape_next(spider, slot)
+ self._scrape_next(spider)
return dfd
- def _scrape_next(self, spider, slot):
- while slot.queue:
- response, request, deferred = slot.next_response_request_deferred()
+ def _scrape_next(self, spider: Spider) -> None:
+ assert self.slot is not None # typing
+ while self.slot.queue:
+ response, request, deferred = self.slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
- def _scrape(self, response, request, spider):
- """Handle the downloaded response or failure through the spider
- callback/errback"""
- if not isinstance(response, (Response, Failure)):
- raise TypeError(
- "Incorrect type: expected Response or Failure, got %s: %r"
- % (type(response), response)
- )
-
- dfd = self._scrape2(response, request, spider) # returns spider's processed output
- dfd.addErrback(self.handle_spider_error, request, response, spider)
- dfd.addCallback(self.handle_spider_output, request, response, spider)
+ def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+ """
+ Handle the downloaded response or failure through the spider callback/errback
+ """
+ if not isinstance(result, (Response, Failure)):
+ raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
+ dfd = self._scrape2(result, request, spider) # returns spider's processed output
+ dfd.addErrback(self.handle_spider_error, request, result, spider)
+ dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
- def _scrape2(self, request_result, request, spider):
- """Handle the different cases of request's result been a Response or a
- Failure"""
- if not isinstance(request_result, Failure):
- return self.spidermw.scrape_response(
- self.call_spider, request_result, request, spider)
- else:
- dfd = self.call_spider(request_result, request, spider)
- return dfd.addErrback(
- self._log_download_errors, request_result, request, spider)
+ def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+ """
+ Handle the different cases of request's result been a Response or a Failure
+ """
+ if isinstance(result, Response):
+ return self.spidermw.scrape_response(self.call_spider, result, request, spider)
+ else: # result is a Failure
+ dfd = self.call_spider(result, request, spider)
+ return dfd.addErrback(self._log_download_errors, result, request, spider)
- def call_spider(self, result, request, spider):
- result.request = request
- dfd = defer_result(result)
- callback = request.callback or spider.parse
- warn_on_generator_with_return_value(spider, callback)
- warn_on_generator_with_return_value(spider, request.errback)
- dfd.addCallbacks(callback=callback,
- errback=request.errback,
- callbackKeywords=request.cb_kwargs)
+ def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+ if isinstance(result, Response):
+ if getattr(result, "request", None) is None:
+ result.request = request
+ callback = result.request.callback or spider._parse
+ warn_on_generator_with_return_value(spider, callback)
+ dfd = defer_succeed(result)
+ dfd.addCallback(callback, **result.request.cb_kwargs)
+ else: # result is a Failure
+ result.request = request
+ warn_on_generator_with_return_value(spider, request.errback)
+ dfd = defer_fail(result)
+ dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
- def handle_spider_error(self, _failure, request, response, spider):
+ def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
exc = _failure.value
if isinstance(exc, CloseSpider):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
@@ -173,11 +181,11 @@ class Scraper:
spider=spider
)
self.crawler.stats.inc_value(
- "spider_exceptions/%s" % _failure.value.__class__.__name__,
+ f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=spider
)
- def handle_spider_output(self, result, request, response, spider):
+ def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)
@@ -185,12 +193,14 @@ class Scraper:
request, response, spider)
return dfd
- def _process_spidermw_output(self, output, request, response, spider):
+ def _process_spidermw_output(self, output: Any, request: Request, response: Response,
+ spider: Spider) -> Optional[Deferred]:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""
+ assert self.slot is not None # typing
if isinstance(output, Request):
- self.crawler.engine.crawl(request=output, spider=spider)
+ self.crawler.engine.crawl(request=output)
elif is_item(output):
self.slot.itemproc_size += 1
dfd = self.itemproc.process_item(output, spider)
@@ -205,12 +215,18 @@ class Scraper:
{'request': request, 'typename': typename},
extra={'spider': spider},
)
+ return None
- def _log_download_errors(self, spider_failure, download_failure, request, spider):
+ def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,
+ spider: Spider) -> Union[Failure, None]:
"""Log and silence errors that come from the engine (typically download
- errors that got propagated thru here)
+ errors that got propagated thru here).
+
+ spider_failure: the value passed into the errback of self.call_spider()
+ download_failure: the value passed into _scrape2() from
+ ExecutionEngine._handle_downloader_output() as "result"
"""
- if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
+ if not download_failure.check(IgnoreRequest):
if download_failure.frames:
logkws = self.logformatter.download_error(download_failure, request, spider)
logger.log(
@@ -230,10 +246,12 @@ class Scraper:
if spider_failure is not download_failure:
return spider_failure
+ return None
- def _itemproc_finished(self, output, item, response, spider):
+ def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:
"""ItemProcessor finished for the given ``item`` and returned ``output``
"""
+ assert self.slot is not None # typing
self.slot.itemproc_size -= 1
if isinstance(output, Failure):
ex = output.value
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 35264a92b..7e58521ac 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,25 +4,25 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
from itertools import islice
+from typing import Any, Callable, Generator, Iterable, Union
+from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
+from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
+from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.python import MutableChain
-def _isiterable(possible_iterator):
- return hasattr(possible_iterator, '__iter__')
+ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
-def _fname(f):
- return "{}.{}".format(
- f.__self__.__class__.__name__,
- f.__func__.__name__
- )
+def _isiterable(o) -> bool:
+ return isinstance(o, Iterable)
class SpiderMiddlewareManager(MiddlewareManager):
@@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
def _add_middleware(self, mw):
- super(SpiderMiddlewareManager, self)._add_middleware(mw)
+ super()._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
@@ -44,85 +44,99 @@ class SpiderMiddlewareManager(MiddlewareManager):
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
- def scrape_response(self, scrape_func, response, request, spider):
-
- def process_spider_input(response):
- for method in self.methods['process_spider_input']:
- try:
- result = method(response=response, spider=spider)
- if result is not None:
- msg = "Middleware {} must return None or raise an exception, got {}"
- raise _InvalidOutput(msg.format(_fname(method), type(result)))
- except _InvalidOutput:
- raise
- except Exception:
- return scrape_func(Failure(), request, spider)
- return scrape_func(response, request, spider)
-
- def _evaluate_iterable(iterable, exception_processor_index, recover_to):
+ def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
+ spider: Spider) -> Any:
+ for method in self.methods['process_spider_input']:
try:
- for r in iterable:
- yield r
+ result = method(response=response, spider=spider)
+ if result is not None:
+ msg = (f"Middleware {method.__qualname__} must return None "
+ f"or raise an exception, got {type(result)}")
+ raise _InvalidOutput(msg)
+ except _InvalidOutput:
+ raise
+ except Exception:
+ return scrape_func(Failure(), request, spider)
+ return scrape_func(response, request, spider)
+
+ def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
+ exception_processor_index: int, recover_to: MutableChain) -> Generator:
+ try:
+ for r in iterable:
+ yield r
+ except Exception as ex:
+ exception_result = self._process_spider_exception(response, spider, Failure(ex),
+ exception_processor_index)
+ if isinstance(exception_result, Failure):
+ raise
+ recover_to.extend(exception_result)
+
+ def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
+ start_index: int = 0) -> Union[Failure, MutableChain]:
+ exception = _failure.value
+ # don't handle _InvalidOutput exception
+ if isinstance(exception, _InvalidOutput):
+ return _failure
+ method_list = islice(self.methods['process_spider_exception'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
+ result = method(response=response, exception=exception, spider=spider)
+ if _isiterable(result):
+ # stop exception handling by handing control over to the
+ # process_spider_output chain if an iterable has been returned
+ return self._process_spider_output(response, spider, result, method_index + 1)
+ elif result is None:
+ continue
+ else:
+ msg = (f"Middleware {method.__qualname__} must return None "
+ f"or an iterable, got {type(result)}")
+ raise _InvalidOutput(msg)
+ return _failure
+
+ def _process_spider_output(self, response: Response, spider: Spider,
+ result: Iterable, start_index: int = 0) -> MutableChain:
+ # items in this iterable do not need to go through the process_spider_output
+ # chain, they went through it already from the process_spider_exception method
+ recovered = MutableChain()
+
+ method_list = islice(self.methods['process_spider_output'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
+ try:
+ # might fail directly if the output value is not a generator
+ result = method(response=response, result=result, spider=spider)
except Exception as ex:
- exception_result = process_spider_exception(Failure(ex), exception_processor_index)
+ exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)
if isinstance(exception_result, Failure):
raise
- recover_to.extend(exception_result)
+ return exception_result
+ if _isiterable(result):
+ result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
+ else:
+ msg = (f"Middleware {method.__qualname__} must return an "
+ f"iterable, got {type(result)}")
+ raise _InvalidOutput(msg)
- def process_spider_exception(_failure, start_index=0):
- exception = _failure.value
- # don't handle _InvalidOutput exception
- if isinstance(exception, _InvalidOutput):
- return _failure
- method_list = islice(self.methods['process_spider_exception'], start_index, None)
- for method_index, method in enumerate(method_list, start=start_index):
- if method is None:
- continue
- result = method(response=response, exception=exception, spider=spider)
- if _isiterable(result):
- # stop exception handling by handing control over to the
- # process_spider_output chain if an iterable has been returned
- return process_spider_output(result, method_index + 1)
- elif result is None:
- continue
- else:
- msg = "Middleware {} must return None or an iterable, got {}"
- raise _InvalidOutput(msg.format(_fname(method), type(result)))
- return _failure
+ return MutableChain(result, recovered)
- def process_spider_output(result, start_index=0):
- # items in this iterable do not need to go through the process_spider_output
- # chain, they went through it already from the process_spider_exception method
- recovered = MutableChain()
+ def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
+ recovered = MutableChain()
+ result = self._evaluate_iterable(response, spider, result, 0, recovered)
+ return MutableChain(self._process_spider_output(response, spider, result), recovered)
- method_list = islice(self.methods['process_spider_output'], start_index, None)
- for method_index, method in enumerate(method_list, start=start_index):
- if method is None:
- continue
- try:
- # might fail directly if the output value is not a generator
- result = method(response=response, result=result, spider=spider)
- except Exception as ex:
- exception_result = process_spider_exception(Failure(ex), method_index + 1)
- if isinstance(exception_result, Failure):
- raise
- return exception_result
- if _isiterable(result):
- result = _evaluate_iterable(result, method_index + 1, recovered)
- else:
- msg = "Middleware {} must return an iterable, got {}"
- raise _InvalidOutput(msg.format(_fname(method), type(result)))
+ def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
+ spider: Spider) -> Deferred:
+ def process_callback_output(result: Iterable) -> MutableChain:
+ return self._process_callback_output(response, spider, result)
- return MutableChain(result, recovered)
+ def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
+ return self._process_spider_exception(response, spider, _failure)
- def process_callback_output(result):
- recovered = MutableChain()
- result = _evaluate_iterable(result, 0, recovered)
- return MutableChain(process_spider_output(result), recovered)
-
- dfd = mustbe_deferred(process_spider_input, response)
+ dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
return dfd
- def process_start_requests(self, start_requests, spider):
+ def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
return self._process_chain('process_start_requests', start_requests, spider)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 6f43771e2..578016536 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -180,9 +180,9 @@ class CrawlerRunner:
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
:class:`~scrapy.spiders.Spider` subclass or string
- :param list args: arguments to initialize the spider
+ :param args: arguments to initialize the spider
- :param dict kwargs: keyword arguments to initialize the spider
+ :param kwargs: keyword arguments to initialize the spider
"""
if isinstance(crawler_or_spidercls, Spider):
raise ValueError(
@@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
"""
def __init__(self, settings=None, install_root_handler=True):
- super(CrawlerProcess, self).__init__(settings)
+ super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@@ -307,7 +307,7 @@ class CrawlerProcess(CrawlerRunner):
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
- :param boolean stop_after_crawl: stop or not the reactor when all
+ :param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
from twisted.internet import reactor
@@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner):
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
- install_reactor(self.settings["TWISTED_REACTOR"])
+ install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
super()._handle_twisted_reactor()
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 77048f389..d95ed3d38 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -54,8 +54,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
- cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
- msg = "Sending cookies to: {}\n{}".format(request, cookies)
+ cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
+ msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _debug_set_cookie(self, response, spider):
@@ -63,8 +63,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
- cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
- msg = "Received cookies from: {}\n{}".format(response, cookies)
+ cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
+ msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie, request):
@@ -74,7 +74,7 @@ class CookiesMiddleware:
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
- if not cookie.get(key):
+ if cookie.get(key) is None:
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
@@ -90,42 +90,21 @@ class CookiesMiddleware:
request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace")
- cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
+ cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
for key, value in decoded.items(): # path, domain
- cookie_str += "; {}={}".format(key.capitalize(), value)
+ cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
def _get_request_cookies(self, jar, request):
"""
- Extract cookies from a Request. Values from the `Request.cookies` attribute
- take precedence over values from the `Cookie` request header.
+ Extract cookies from the Request.cookies attribute
"""
- def get_cookies_from_header(jar, request):
- cookie_header = request.headers.get("Cookie")
- if not cookie_header:
- return []
- cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
- cookie_list_unicode = []
- for cookie_bytes in cookie_gen_bytes:
- try:
- cookie_unicode = cookie_bytes.decode("utf8")
- except UnicodeDecodeError:
- logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
- request, cookie_bytes)
- cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
- cookie_list_unicode.append(cookie_unicode)
- response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
- return jar.make_cookies(response, request)
-
- def get_cookies_from_attribute(jar, request):
- if not request.cookies:
- return []
- elif isinstance(request.cookies, dict):
- cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
- else:
- cookies = request.cookies
- formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
- response = Response(request.url, headers={"Set-Cookie": formatted})
- return jar.make_cookies(response, request)
-
- return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
+ if not request.cookies:
+ return []
+ elif isinstance(request.cookies, dict):
+ cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
+ else:
+ cookies = request.cookies
+ formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
+ response = Response(request.url, headers={"Set-Cookie": formatted})
+ return jar.make_cookies(response, request)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 6db57bd8b..62f1c3a29 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,4 +1,5 @@
from email.utils import formatdate
+from typing import Optional, Type, TypeVar
from twisted.internet import defer
from twisted.internet.error import (
@@ -13,10 +14,19 @@ from twisted.internet.error import (
from twisted.web.client import ResponseFailed
from scrapy import signals
+from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.http.request import Request
+from scrapy.http.response import Response
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
+HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
+
+
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@@ -24,7 +34,7 @@ class HttpCacheMiddleware:
ConnectionLost, TCPTimedOutError, ResponseFailed,
IOError)
- def __init__(self, settings, stats):
+ def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
@@ -33,26 +43,26 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
- def from_crawler(cls, crawler):
+ def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
- def spider_opened(self, spider):
+ def spider_opened(self, spider: Spider) -> None:
self.storage.open_spider(spider)
- def spider_closed(self, spider):
+ def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
- def process_request(self, request, spider):
+ def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
if request.meta.get('dont_cache', False):
- return
+ return None
# Skip uncacheable requests
if not self.policy.should_cache_request(request):
request.meta['_dont_cache'] = True # flag as uncacheable
- return
+ return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
@@ -61,7 +71,7 @@ class HttpCacheMiddleware:
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
raise IgnoreRequest("Ignored request not in cache: %s" % request)
- return # first time request
+ return None # first time request
# Return cached response only if not expired
cachedresponse.flags.append('cached')
@@ -73,7 +83,9 @@ class HttpCacheMiddleware:
# process_response hook
request.meta['cached_response'] = cachedresponse
- def process_response(self, request, response, spider):
+ return None
+
+ def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
if request.meta.get('dont_cache', False):
return response
@@ -85,7 +97,7 @@ class HttpCacheMiddleware:
# RFC2616 requires origin server to set Date header,
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
if 'Date' not in response.headers:
- response.headers['Date'] = formatdate(usegmt=1)
+ response.headers['Date'] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop('cached_response', None)
@@ -102,13 +114,18 @@ class HttpCacheMiddleware:
self._cache_response(spider, response, request, cachedresponse)
return response
- def process_exception(self, request, exception, spider):
+ def process_exception(
+ self, request: Request, exception: Exception, spider: Spider
+ ) -> Optional[Response]:
cachedresponse = request.meta.pop('cached_response', None)
if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
self.stats.inc_value('httpcache/errorrecovery', spider=spider)
return cachedresponse
+ return None
- def _cache_response(self, spider, response, request, cachedresponse):
+ def _cache_response(
+ self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
+ ) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value('httpcache/store', spider=spider)
self.storage.store_response(spider, request, response)
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 727c41466..4e7feeeaf 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,9 +1,12 @@
+import io
+import warnings
import zlib
-from scrapy.utils.gz import gunzip
+from scrapy.exceptions import NotConfigured
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
-from scrapy.exceptions import NotConfigured
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.gz import gunzip
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
@@ -14,15 +17,35 @@ try:
except ImportError:
pass
+try:
+ import zstandard
+ ACCEPTED_ENCODINGS.append(b'zstd')
+except ImportError:
+ pass
+
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites"""
+ def __init__(self, stats=None):
+ self.stats = stats
+
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('COMPRESSION_ENABLED'):
raise NotConfigured
- return cls()
+ try:
+ return cls(stats=crawler.stats)
+ except TypeError:
+ warnings.warn(
+ "HttpCompressionMiddleware subclasses must either modify "
+ "their '__init__' method to support a 'stats' parameter or "
+ "reimplement the 'from_crawler' method.",
+ ScrapyDeprecationWarning,
+ )
+ result = cls()
+ result.stats = crawler.stats
+ return result
def process_request(self, request, spider):
request.headers.setdefault('Accept-Encoding',
@@ -37,6 +60,9 @@ class HttpCompressionMiddleware:
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
+ if self.stats:
+ self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+ self.stats.inc_value('httpcompression/response_count', spider=spider)
respcls = responsetypes.from_args(
headers=response.headers, url=response.url, body=decoded_body
)
@@ -67,4 +93,9 @@ class HttpCompressionMiddleware:
body = zlib.decompress(body, -15)
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
+ if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
+ # Using its streaming API since its simple API could handle only cases
+ # where there is content size data embedded in the frame
+ reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
+ body = reader.read()
return body
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index da89d3e9b..d2665b655 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -13,7 +13,12 @@ class HttpProxyMiddleware:
self.auth_encoding = auth_encoding
self.proxies = {}
for type_, url in getproxies().items():
- self.proxies[type_] = self._get_proxy(url, type_)
+ try:
+ self.proxies[type_] = self._get_proxy(url, type_)
+ # some values such as '/var/run/docker.sock' can't be parsed
+ # by _parse_proxy and as such should be skipped
+ except ValueError:
+ continue
@classmethod
def from_crawler(cls, crawler):
@@ -24,7 +29,7 @@ class HttpProxyMiddleware:
def _basic_auth_header(self, username, password):
user_pass = to_bytes(
- '%s:%s' % (unquote(username), unquote(password)),
+ f'{unquote(username)}:{unquote(password)}',
encoding=self.auth_encoding)
return base64.b64encode(user_pass)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b32afb8e4..4053fecc5 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
if ttl and redirects <= self.max_redirect_times:
redirected.meta['redirect_times'] = redirects
redirected.meta['redirect_ttl'] = ttl - 1
- redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
- [request.url]
- redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
- [reason]
+ redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
+ redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -94,13 +92,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = 'METAREFRESH_ENABLED'
def __init__(self, settings):
- super(MetaRefreshMiddleware, self).__init__(settings)
+ super().__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
def process_response(self, request, response, spider):
- if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
- not isinstance(response, HtmlResponse):
+ if (
+ request.meta.get('dont_redirect', False)
+ or request.method == 'HEAD'
+ or not isinstance(response, HtmlResponse)
+ ):
return response
interval, url = get_meta_refresh(response,
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 6d11af5b2..f1fdc3858 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,7 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
-import logging
+from logging import getLogger, Logger
+from typing import Optional, Union
from twisted.internet import defer
from twisted.internet.error import (
@@ -23,12 +24,104 @@ from twisted.internet.error import (
)
from twisted.web.client import ResponseFailed
-from scrapy.exceptions import NotConfigured
-from scrapy.utils.response import response_status_message
from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
from scrapy.utils.python import global_object_name
+from scrapy.utils.response import response_status_message
-logger = logging.getLogger(__name__)
+
+retry_logger = getLogger(__name__)
+
+
+def get_retry_request(
+ request: Request,
+ *,
+ spider: Spider,
+ reason: Union[str, Exception] = 'unspecified',
+ max_retry_times: Optional[int] = None,
+ priority_adjust: Optional[int] = None,
+ logger: Logger = retry_logger,
+ stats_base_key: str = 'retry',
+):
+ """
+ Returns a new :class:`~scrapy.Request` object to retry the specified
+ request, or ``None`` if retries of the specified request have been
+ exhausted.
+
+ For example, in a :class:`~scrapy.Spider` callback, you could use it as
+ follows::
+
+ def parse(self, response):
+ if not response.text:
+ new_request_or_none = get_retry_request(
+ response.request,
+ spider=self,
+ reason='empty',
+ )
+ return new_request_or_none
+
+ *spider* is the :class:`~scrapy.Spider` instance which is asking for the
+ retry request. It is used to access the :ref:`settings `
+ and :ref:`stats `, and to provide extra logging context (see
+ :func:`logging.debug`).
+
+ *reason* is a string or an :class:`Exception` object that indicates the
+ reason why the request needs to be retried. It is used to name retry stats.
+
+ *max_retry_times* is a number that determines the maximum number of times
+ that *request* can be retried. If not specified or ``None``, the number is
+ read from the :reqmeta:`max_retry_times` meta key of the request. If the
+ :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
+ is read from the :setting:`RETRY_TIMES` setting.
+
+ *priority_adjust* is a number that determines how the priority of the new
+ request changes in relation to *request*. If not specified, the number is
+ read from the :setting:`RETRY_PRIORITY_ADJUST` setting.
+
+ *logger* is the logging.Logger object to be used when logging messages
+
+ *stats_base_key* is a string to be used as the base key for the
+ retry-related job stats
+ """
+ settings = spider.crawler.settings
+ stats = spider.crawler.stats
+ retry_times = request.meta.get('retry_times', 0) + 1
+ if max_retry_times is None:
+ max_retry_times = request.meta.get('max_retry_times')
+ if max_retry_times is None:
+ max_retry_times = settings.getint('RETRY_TIMES')
+ if retry_times <= max_retry_times:
+ logger.debug(
+ "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
+ {'request': request, 'retry_times': retry_times, 'reason': reason},
+ extra={'spider': spider}
+ )
+ new_request: Request = request.copy()
+ new_request.meta['retry_times'] = retry_times
+ new_request.dont_filter = True
+ if priority_adjust is None:
+ priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+ new_request.priority = request.priority + priority_adjust
+
+ if callable(reason):
+ reason = reason()
+ if isinstance(reason, Exception):
+ reason = global_object_name(reason.__class__)
+
+ stats.inc_value(f'{stats_base_key}/count')
+ stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
+ return new_request
+ else:
+ stats.inc_value(f'{stats_base_key}/max_reached')
+ logger.error(
+ "Gave up retrying %(request)s (failed %(retry_times)d times): "
+ "%(reason)s",
+ {'request': request, 'retry_times': retry_times, 'reason': reason},
+ extra={'spider': spider},
+ )
+ return None
class RetryMiddleware:
@@ -60,36 +153,19 @@ class RetryMiddleware:
return response
def process_exception(self, request, exception, spider):
- if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
- and not request.meta.get('dont_retry', False):
+ if (
+ isinstance(exception, self.EXCEPTIONS_TO_RETRY)
+ and not request.meta.get('dont_retry', False)
+ ):
return self._retry(request, exception, spider)
def _retry(self, request, reason, spider):
- retries = request.meta.get('retry_times', 0) + 1
-
- retry_times = self.max_retry_times
-
- if 'max_retry_times' in request.meta:
- retry_times = request.meta['max_retry_times']
-
- stats = spider.crawler.stats
- if retries <= retry_times:
- logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
- {'request': request, 'retries': retries, 'reason': reason},
- extra={'spider': spider})
- retryreq = request.copy()
- retryreq.meta['retry_times'] = retries
- retryreq.dont_filter = True
- retryreq.priority = request.priority + self.priority_adjust
-
- if isinstance(reason, Exception):
- reason = global_object_name(reason.__class__)
-
- stats.inc_value('retry/count')
- stats.inc_value('retry/reason_count/%s' % reason)
- return retryreq
- else:
- stats.inc_value('retry/max_reached')
- logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
- {'request': request, 'retries': retries, 'reason': reason},
- extra={'spider': spider})
+ max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
+ priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
+ return get_retry_request(
+ request,
+ reason=reason,
+ spider=spider,
+ max_retry_times=max_retry_times,
+ priority_adjust=priority_adjust,
+ )
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 7f18b2bf2..e66bf177e 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -61,13 +61,13 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers:
self._parsers[netloc] = Deferred()
- robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
+ robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
meta={'dont_obey_robotstxt': True}
)
- dfd = self.crawler.engine.download(robotsreq, spider)
+ dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
@@ -94,7 +94,7 @@ class RobotsTxtMiddleware:
def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
- self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+ self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
@@ -102,7 +102,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, failure, netloc):
if failure.type is not IgnoreRequest:
- key = 'robotstxt/exception_count/{}'.format(failure.type)
+ key = f'robotstxt/exception_count/{failure.type}'
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = None
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 46a2ad397..5479cd0e2 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -17,13 +17,13 @@ class DownloaderStats:
def process_request(self, request, spider):
self.stats.inc_value('downloader/request_count', spider=spider)
- self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
+ self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
reqlen = len(request_httprepr(request))
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider)
- self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
+ self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response_httprepr(response))
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response
@@ -31,4 +31,4 @@ class DownloaderStats:
def process_exception(self, request, exception, spider):
ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider)
- self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
+ self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index ac5478e7c..292c68099 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,35 +1,47 @@
-import os
import logging
+import os
+from typing import Optional, Set, Type, TypeVar
+from twisted.internet.defer import Deferred
+
+from scrapy.http.request import Request
+from scrapy.settings import BaseSettings
+from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, request_fingerprint
-class BaseDupeFilter:
+BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
+
+class BaseDupeFilter:
@classmethod
- def from_settings(cls, settings):
+ def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV:
return cls()
- def request_seen(self, request):
+ def request_seen(self, request: Request) -> bool:
return False
- def open(self): # can return deferred
+ def open(self) -> Optional[Deferred]:
pass
- def close(self, reason): # can return a deferred
+ def close(self, reason: str) -> Optional[Deferred]:
pass
- def log(self, request, spider): # log that a request has been filtered
+ def log(self, request: Request, spider: Spider) -> None:
+ """Log that a request has been filtered"""
pass
+RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
+
+
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
- def __init__(self, path=None, debug=False):
+ def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
self.file = None
- self.fingerprints = set()
+ self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
self.logger = logging.getLogger(__name__)
@@ -39,26 +51,27 @@ class RFPDupeFilter(BaseDupeFilter):
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
- def from_settings(cls, settings):
+ def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
return cls(job_dir(settings), debug)
- def request_seen(self, request):
+ def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
if fp in self.fingerprints:
return True
self.fingerprints.add(fp)
if self.file:
self.file.write(fp + '\n')
+ return False
- def request_fingerprint(self, request):
+ def request_fingerprint(self, request: Request) -> str:
return request_fingerprint(request)
- def close(self, reason):
+ def close(self, reason: str) -> None:
if self.file:
self.file.close()
- def log(self, request, spider):
+ def log(self, request: Request, spider: Spider) -> None:
if self.debug:
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
args = {'request': request, 'referer': referer_str(request)}
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 45f152321..0c410f035 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -37,7 +37,7 @@ class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason='cancelled'):
- super(CloseSpider, self).__init__()
+ super().__init__()
self.reason = reason
@@ -74,7 +74,7 @@ class UsageError(Exception):
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
- super(UsageError, self).__init__(*a, **kw)
+ super().__init__(*a, **kw)
class ScrapyDeprecationWarning(Warning):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 712572673..fb4b565cf 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -39,7 +39,7 @@ class BaseItemExporter:
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None)
if not dont_fail and options:
- raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
+ raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
def export_item(self, item):
raise NotImplementedError
@@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
- def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
+ def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
@@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
- newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ errors=errors,
)
self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True
@@ -243,12 +244,8 @@ class CsvItemExporter(BaseItemExporter):
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
- if isinstance(item, dict):
- # for dicts try using fields of the first item
- self.fields_to_export = list(item.keys())
- else:
- # use fields declared in Item
- self.fields_to_export = list(item.fields.keys())
+ # use declared field names, or keys if the item is a dict
+ self.fields_to_export = ItemAdapter(item).field_names()
row = list(self._build_row(self.fields_to_export))
self.csv_writer.writerow(row)
@@ -305,7 +302,7 @@ class PythonItemExporter(BaseItemExporter):
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
- super(PythonItemExporter, self)._configure(options, dont_fail)
+ super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 389cb65bc..675f8276f 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -43,4 +43,4 @@ class CoreStats:
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
self.stats.inc_value('item_dropped_count', spider=spider)
- self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
+ self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 586399784..fd2a02d8d 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -48,7 +48,7 @@ class StackTraceDump:
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, '')
dump = ''.join(traceback.format_stack(frame))
- dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
+ dumps += f"# Thread: {name}({id_})\n{dump}\n"
return dumps
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 61dad8726..bec114707 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
import logging
import os
+import re
import sys
import warnings
from datetime import datetime
@@ -18,22 +19,39 @@ from zope.interface import implementer, Interface
from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.python import without_none_values
+from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
+def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
+ argument_names = get_func_args(builder)
+ if 'feed_options' in argument_names:
+ kwargs['feed_options'] = feed_options
+ else:
+ warnings.warn(
+ "{} does not support the 'feed_options' keyword argument. Add a "
+ "'feed_options' parameter to its signature to remove this "
+ "warning. This parameter will become mandatory in a future "
+ "version of Scrapy."
+ .format(builder.__qualname__),
+ category=ScrapyDeprecationWarning
+ )
+ return builder(*preargs, uri, *args, **kwargs)
+
+
class IFeedStorage(Interface):
"""Interface that all Feed Storages must implement"""
- def __init__(uri):
- """Initialize the storage with the parameters given in the URI"""
+ def __init__(uri, *, feed_options=None):
+ """Initialize the storage with the parameters given in the URI and the
+ feed-specific options (see :setting:`FEEDS`)"""
def open(spider):
"""Open the storage for the given spider. It must return a file-like
@@ -63,10 +81,15 @@ class BlockingFeedStorage:
@implementer(IFeedStorage)
class StdoutFeedStorage:
- def __init__(self, uri, _stdout=None):
+ def __init__(self, uri, _stdout=None, *, feed_options=None):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
+ if feed_options and feed_options.get('overwrite', False) is True:
+ logger.warning('Standard output (stdout) storage does not support '
+ 'overwriting. To suppress this warning, remove the '
+ 'overwrite option from your FEEDS setting, or set '
+ 'it to False.')
def open(self, spider):
return self._stdout
@@ -78,14 +101,16 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
class FileFeedStorage:
- def __init__(self, uri):
+ def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
+ feed_options = feed_options or {}
+ self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
- return open(self.path, 'ab')
+ return open(self.path, self.write_mode)
def store(self, file):
file.close()
@@ -93,101 +118,113 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
- def __init__(self, uri, access_key=None, secret_key=None, acl=None):
- # BEGIN Backward compatibility for initialising without keys (and
- # without using from_crawler)
- no_defaults = access_key is None and secret_key is None
- if no_defaults:
- from scrapy.utils.project import get_project_settings
- settings = get_project_settings()
- if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
- warnings.warn(
- "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
- "without AWS keys is deprecated. Please supply credentials or "
- "use the `from_crawler()` constructor.",
- category=ScrapyDeprecationWarning,
- stacklevel=2
- )
- access_key = settings['AWS_ACCESS_KEY_ID']
- secret_key = settings['AWS_SECRET_ACCESS_KEY']
- # END Backward compatibility
+ def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+ feed_options=None):
+ if not is_botocore_available():
+ raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
- self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
- if self.is_botocore:
- import botocore.session
- session = botocore.session.get_session()
- self.s3_client = session.create_client(
- 's3', aws_access_key_id=self.access_key,
- aws_secret_access_key=self.secret_key)
- else:
- import boto
- self.connect_s3 = boto.connect_s3
+ import botocore.session
+ session = botocore.session.get_session()
+ self.s3_client = session.create_client(
+ 's3', aws_access_key_id=self.access_key,
+ aws_secret_access_key=self.secret_key)
+ if feed_options and feed_options.get('overwrite', True) is False:
+ logger.warning('S3 does not support appending to files. To '
+ 'suppress this warning, remove the overwrite '
+ 'option from your FEEDS setting or set it to True.')
@classmethod
- def from_crawler(cls, crawler, uri):
- return cls(
- uri=uri,
+ def from_crawler(cls, crawler, uri, *, feed_options=None):
+ return build_storage(
+ cls,
+ uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
- acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
+ acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+ feed_options=feed_options,
)
def _store_in_thread(self, file):
file.seek(0)
- if self.is_botocore:
- kwargs = {'ACL': self.acl} if self.acl else {}
- self.s3_client.put_object(
- Bucket=self.bucketname, Key=self.keyname, Body=file,
- **kwargs)
- else:
- conn = self.connect_s3(self.access_key, self.secret_key)
- bucket = conn.get_bucket(self.bucketname, validate=False)
- key = bucket.new_key(self.keyname)
- kwargs = {'policy': self.acl} if self.acl else {}
- key.set_contents_from_file(file, **kwargs)
- key.close()
+ kwargs = {'ACL': self.acl} if self.acl else {}
+ self.s3_client.put_object(
+ Bucket=self.bucketname, Key=self.keyname, Body=file,
+ **kwargs)
+ file.close()
-class FTPFeedStorage(BlockingFeedStorage):
+class GCSFeedStorage(BlockingFeedStorage):
- def __init__(self, uri, use_active_mode=False):
+ def __init__(self, uri, project_id, acl):
+ self.project_id = project_id
+ self.acl = acl
u = urlparse(uri)
- self.host = u.hostname
- self.port = int(u.port or '21')
- self.username = u.username
- self.password = unquote(u.password)
- self.path = u.path
- self.use_active_mode = use_active_mode
+ self.bucket_name = u.hostname
+ self.blob_name = u.path[1:] # remove first "/"
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
- uri=uri,
- use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+ uri,
+ crawler.settings['GCS_PROJECT_ID'],
+ crawler.settings['FEED_STORAGE_GCS_ACL'] or None
+ )
+
+ def _store_in_thread(self, file):
+ file.seek(0)
+ from google.cloud.storage import Client
+ client = Client(project=self.project_id)
+ bucket = client.get_bucket(self.bucket_name)
+ blob = bucket.blob(self.blob_name)
+ blob.upload_from_file(file, predefined_acl=self.acl)
+
+
+class FTPFeedStorage(BlockingFeedStorage):
+
+ def __init__(self, uri, use_active_mode=False, *, feed_options=None):
+ u = urlparse(uri)
+ self.host = u.hostname
+ self.port = int(u.port or '21')
+ self.username = u.username
+ self.password = unquote(u.password or '')
+ self.path = u.path
+ self.use_active_mode = use_active_mode
+ self.overwrite = not feed_options or feed_options.get('overwrite', True)
+
+ @classmethod
+ def from_crawler(cls, crawler, uri, *, feed_options=None):
+ return build_storage(
+ cls,
+ uri,
+ crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
+ feed_options=feed_options,
)
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
port=self.port, username=self.username,
- password=self.password, use_active_mode=self.use_active_mode
+ password=self.password, use_active_mode=self.use_active_mode,
+ overwrite=self.overwrite,
)
class _FeedSlot:
- def __init__(self, file, exporter, storage, uri, format, store_empty):
+ def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
self.file = file
self.exporter = exporter
self.storage = storage
# feed params
- self.uri = uri
+ self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
+ self.uri_template = uri_template
+ self.uri = uri
# flags
self.itemcount = 0
self._exporting = False
@@ -230,77 +267,135 @@ class FeedExporter:
category=ScrapyDeprecationWarning, stacklevel=2,
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
- feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+ feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
- for uri, feed in self.settings.getdict('FEEDS').items():
+ for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
- self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
+ self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
- for uri, feed in self.feeds.items():
- if not self._storage_supported(uri):
+ for uri, feed_options in self.feeds.items():
+ if not self._storage_supported(uri, feed_options):
raise NotConfigured
- if not self._exporter_supported(feed['format']):
+ if not self._settings_are_valid():
+ raise NotConfigured
+ if not self._exporter_supported(feed_options['format']):
raise NotConfigured
def open_spider(self, spider):
- for uri, feed in self.feeds.items():
- uri = uri % self._get_uri_params(spider, feed['uri_params'])
- storage = self._get_storage(uri)
- file = storage.open(spider)
- exporter = self._get_exporter(
- file=file,
- format=feed['format'],
- fields_to_export=feed['fields'],
- encoding=feed['encoding'],
- indent=feed['indent'],
- )
- slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
- self.slots.append(slot)
- if slot.store_empty:
- slot.start_exporting()
+ for uri, feed_options in self.feeds.items():
+ uri_params = self._get_uri_params(spider, feed_options['uri_params'])
+ self.slots.append(self._start_new_batch(
+ batch_id=1,
+ uri=uri % uri_params,
+ feed_options=feed_options,
+ spider=spider,
+ uri_template=uri,
+ ))
def close_spider(self, spider):
deferred_list = []
for slot in self.slots:
- if not slot.itemcount and not slot.store_empty:
- # We need to call slot.storage.store nonetheless to get the file
- # properly closed.
- d = defer.maybeDeferred(slot.storage.store, slot.file)
- deferred_list.append(d)
- continue
- slot.finish_exporting()
- logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
- log_args = {'format': slot.format,
- 'itemcount': slot.itemcount,
- 'uri': slot.uri}
- d = defer.maybeDeferred(slot.storage.store, slot.file)
-
- # Use `largs=log_args` to copy log_args into function's scope
- # instead of using `log_args` from the outer scope
- d.addCallback(
- lambda _, largs=log_args: logger.info(
- logfmt % "Stored", largs, extra={'spider': spider}
- )
- )
- d.addErrback(
- lambda f, largs=log_args: logger.error(
- logfmt % "Error storing", largs,
- exc_info=failure_to_exc_info(f), extra={'spider': spider}
- )
- )
+ d = self._close_slot(slot, spider)
deferred_list.append(d)
return defer.DeferredList(deferred_list) if deferred_list else None
+ def _close_slot(self, slot, spider):
+ if not slot.itemcount and not slot.store_empty:
+ # We need to call slot.storage.store nonetheless to get the file
+ # properly closed.
+ return defer.maybeDeferred(slot.storage.store, slot.file)
+ slot.finish_exporting()
+ logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
+ log_args = {'format': slot.format,
+ 'itemcount': slot.itemcount,
+ 'uri': slot.uri}
+ d = defer.maybeDeferred(slot.storage.store, slot.file)
+
+ # Use `largs=log_args` to copy log_args into function's scope
+ # instead of using `log_args` from the outer scope
+ d.addCallback(
+ self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
+ )
+ d.addErrback(
+ self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
+ )
+ return d
+
+ def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
+ logger.error(
+ logfmt % "Error storing", largs,
+ exc_info=failure_to_exc_info(f), extra={'spider': spider}
+ )
+ self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
+
+ def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
+ logger.info(
+ logfmt % "Stored", largs, extra={'spider': spider}
+ )
+ self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
+
+ def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
+ """
+ Redirect the output data stream to a new file.
+ Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
+ :param batch_id: sequence number of current batch
+ :param uri: uri of the new batch to start
+ :param feed_options: dict with parameters of feed
+ :param spider: user spider
+ :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
+ """
+ storage = self._get_storage(uri, feed_options)
+ file = storage.open(spider)
+ exporter = self._get_exporter(
+ file=file,
+ format=feed_options['format'],
+ fields_to_export=feed_options['fields'],
+ encoding=feed_options['encoding'],
+ indent=feed_options['indent'],
+ **feed_options['item_export_kwargs'],
+ )
+ slot = _FeedSlot(
+ file=file,
+ exporter=exporter,
+ storage=storage,
+ uri=uri,
+ format=feed_options['format'],
+ store_empty=feed_options['store_empty'],
+ batch_id=batch_id,
+ uri_template=uri_template,
+ )
+ if slot.store_empty:
+ slot.start_exporting()
+ return slot
+
def item_scraped(self, item, spider):
+ slots = []
for slot in self.slots:
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
+ # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
+ if (
+ self.feeds[slot.uri_template]['batch_item_count']
+ and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+ ):
+ uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
+ self._close_slot(slot, spider)
+ slots.append(self._start_new_batch(
+ batch_id=slot.batch_id + 1,
+ uri=slot.uri_template % uri_params,
+ feed_options=self.feeds[slot.uri_template],
+ spider=spider,
+ uri_template=slot.uri_template,
+ ))
+ else:
+ slots.append(slot)
+ self.slots = slots
def _load_components(self, setting_prefix):
conf = without_none_values(self.settings.getwithbase(setting_prefix))
@@ -317,11 +412,27 @@ class FeedExporter:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
- def _storage_supported(self, uri):
+ def _settings_are_valid(self):
+ """
+ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+ %(batch_time)s or %(batch_id)d to distinguish different files of partial output
+ """
+ for uri_template, values in self.feeds.items():
+ if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
+ logger.error(
+ '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+ 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
+ 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
+ ''.format(uri_template)
+ )
+ return False
+ return True
+
+ def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
try:
- self._get_storage(uri)
+ self._get_storage(uri, feed_options)
return True
except NotConfigured as e:
logger.error("Disabled feed storage scheme: %(scheme)s. "
@@ -339,15 +450,39 @@ class FeedExporter:
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
- def _get_storage(self, uri):
- return self._get_instance(self.storages[urlparse(uri).scheme], uri)
+ def _get_storage(self, uri, feed_options):
+ """Fork of create_instance specific to feed storage classes
- def _get_uri_params(self, spider, uri_params):
+ It supports not passing the *feed_options* parameters to classes that
+ do not support it, and issuing a deprecation warning instead.
+ """
+ feedcls = self.storages[urlparse(uri).scheme]
+ crawler = getattr(self, 'crawler', None)
+
+ def build_instance(builder, *preargs):
+ return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
+
+ if crawler and hasattr(feedcls, 'from_crawler'):
+ instance = build_instance(feedcls.from_crawler, crawler)
+ method_name = 'from_crawler'
+ elif hasattr(feedcls, 'from_settings'):
+ instance = build_instance(feedcls.from_settings, self.settings)
+ method_name = 'from_settings'
+ else:
+ instance = build_instance(feedcls)
+ method_name = '__new__'
+ if instance is None:
+ raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
+ return instance
+
+ def _get_uri_params(self, spider, uri_params, slot=None):
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
- ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
- params['time'] = ts
+ utc_now = datetime.utcnow()
+ params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
+ params['batch_time'] = utc_now.isoformat().replace(':', '-')
+ params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
uripar_function(params, spider)
return params
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 6294a9b52..e0c04b2de 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -223,7 +223,7 @@ class DbmCacheStorage:
self.db = None
def open_spider(self, spider):
- dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
+ dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
@@ -251,13 +251,13 @@ class DbmCacheStorage:
'headers': dict(response.headers),
'body': response.body,
}
- self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
- self.db['%s_time' % key] = str(time())
+ self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
+ self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
key = self._request_key(request)
db = self.db
- tkey = '%s_time' % key
+ tkey = f'{key}_time'
if tkey not in db:
return # not found
@@ -265,7 +265,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return # expired
- return pickle.loads(db['%s_data' % key])
+ return pickle.loads(db[f'{key}_data'])
def _request_key(self, request):
return request_fingerprint(request)
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index dc8cdbb1d..cee44ea62 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -30,4 +30,4 @@ class MemoryDebugger:
for cls, wdict in live_refs.items():
if not wdict:
continue
- self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
+ self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index a0540bf8f..9de119a10 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -81,15 +81,15 @@ class MemoryUsage:
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
- subj = "%s terminated: memory usage exceeded %dM at %s" % \
- (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+ subj = (
+ f"{self.crawler.settings['BOT_NAME']} terminated: "
+ f"memory usage exceeded {mem}M at {socket.gethostname()}"
+ )
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1)
- open_spiders = self.crawler.engine.open_spiders
- if open_spiders:
- for spider in open_spiders:
- self.crawler.engine.close_spider(spider, 'memusage_exceeded')
+ if self.crawler.engine.spider is not None:
+ self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
else:
self.crawler.stop()
@@ -102,8 +102,10 @@ class MemoryUsage:
logger.warning("Memory usage reached %(memusage)dM",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
- subj = "%s warning: memory usage reached %dM at %s" % \
- (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
+ subj = (
+ f"{self.crawler.settings['BOT_NAME']} warning: "
+ f"memory usage reached {mem}M at {socket.gethostname()}"
+ )
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1)
self.warned = True
@@ -111,9 +113,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
- s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
- s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
- s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)
+ s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
+ s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
+ s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 320f13b29..bcdbaff24 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -24,11 +24,11 @@ class StatsMailer:
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
-
+
def spider_closed(self, spider):
spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n"
- body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items())
- body += "\n\n%s stats\n\n" % spider.name
- body += "\n".join("%-50s : %s" % i for i in spider_stats.items())
- return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body)
+ body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
+ body += f"\n\n{spider.name} stats\n\n"
+ body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items())
+ return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body)
diff --git a/scrapy/http/common.py b/scrapy/http/common.py
index ba6ab277c..98699d7fd 100644
--- a/scrapy/http/common.py
+++ b/scrapy/http/common.py
@@ -1,6 +1,6 @@
def obsolete_setter(setter, attrname):
def newsetter(self, value):
c = self.__class__.__name__
- msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c)
+ msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
raise AttributeError(msg)
return newsetter
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 3e810992c..bf4ae7b45 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,10 +1,16 @@
+import re
import time
-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
+# Defined in the http.cookiejar module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
+IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
+
+
class CookieJar:
def __init__(self, policy=None, check_expired_frequency=10000):
self.policy = policy or DefaultCookiePolicy()
@@ -186,9 +192,6 @@ class WrappedResponse:
def info(self):
return self
- # python3 cookiejars calls get_all
def get_all(self, name, default=None):
return [to_unicode(v, errors='replace')
for v in self.response.headers.getlist(name)]
- # python2 cookiejars calls getheaders
- getheaders = get_all
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index dcaaeddfa..1a2b99b0a 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -8,7 +8,7 @@ class Headers(CaselessDict):
def __init__(self, seq=None, encoding='utf-8'):
self.encoding = encoding
- super(Headers, self).__init__(seq)
+ super().__init__(seq)
def normkey(self, key):
"""Normalize key to bytes"""
@@ -33,23 +33,23 @@ class Headers(CaselessDict):
elif isinstance(x, int):
return str(x).encode(self.encoding)
else:
- raise TypeError('Unsupported value type: {}'.format(type(x)))
+ raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key):
try:
- return super(Headers, self).__getitem__(key)[-1]
+ return super().__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
- return super(Headers, self).get(key, def_val)[-1]
+ return super().get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
- return super(Headers, self).__getitem__(key)
+ return super().__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a98ba9960..7672dec00 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,34 +4,69 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
+import inspect
+from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
+
from w3lib.url import safe_url_string
+import scrapy
+from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
+from scrapy.utils.curl import curl_to_request_kwargs
from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
-from scrapy.http.common import obsolete_setter
-from scrapy.utils.curl import curl_to_request_kwargs
+
+
+RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
class Request(object_ref):
+ """Represents an HTTP request, which is usually generated in a Spider and
+ executed by the Downloader, thus generating a :class:`Response`.
+ """
- def __init__(self, url, callback=None, method='GET', headers=None, body=None,
- cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, flags=None, cb_kwargs=None):
+ attributes: Tuple[str, ...] = (
+ "url", "callback", "method", "headers", "body",
+ "cookies", "meta", "encoding", "priority",
+ "dont_filter", "errback", "flags", "cb_kwargs",
+ )
+ """A tuple of :class:`str` objects containing the name of all public
+ attributes of the class that are also keyword parameters of the
+ ``__init__`` method.
+ Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and
+ :func:`~scrapy.utils.request.request_from_dict`.
+ """
+
+ def __init__(
+ self,
+ url: str,
+ callback: Optional[Callable] = None,
+ method: str = "GET",
+ headers: Optional[dict] = None,
+ body: Optional[Union[bytes, str]] = None,
+ cookies: Optional[Union[dict, List[dict]]] = None,
+ meta: Optional[dict] = None,
+ encoding: str = "utf-8",
+ priority: int = 0,
+ dont_filter: bool = False,
+ errback: Optional[Callable] = None,
+ flags: Optional[List[str]] = None,
+ cb_kwargs: Optional[dict] = None,
+ ) -> None:
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
- raise TypeError("Request priority not an integer: %r" % priority)
+ raise TypeError(f"Request priority not an integer: {priority!r}")
self.priority = priority
if callback is not None and not callable(callback):
- raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
+ raise TypeError(f'callback must be a callable, got {type(callback).__name__}')
if errback is not None and not callable(errback):
- raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
+ raise TypeError(f'errback must be a callable, got {type(errback).__name__}')
self.callback = callback
self.errback = errback
@@ -44,68 +79,67 @@ class Request(object_ref):
self.flags = [] if flags is None else list(flags)
@property
- def cb_kwargs(self):
+ def cb_kwargs(self) -> dict:
if self._cb_kwargs is None:
self._cb_kwargs = {}
return self._cb_kwargs
@property
- def meta(self):
+ def meta(self) -> dict:
if self._meta is None:
self._meta = {}
return self._meta
- def _get_url(self):
+ def _get_url(self) -> str:
return self._url
- def _set_url(self, url):
+ def _set_url(self, url: str) -> None:
if not isinstance(url, str):
- raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
+ raise TypeError(f"Request url must be str, got {type(url).__name__}")
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
- if ('://' not in self._url) and (not self._url.startswith('data:')):
- raise ValueError('Missing scheme in request url: %s' % self._url)
+ if (
+ '://' not in self._url
+ and not self._url.startswith('about:')
+ and not self._url.startswith('data:')
+ ):
+ raise ValueError(f'Missing scheme in request url: {self._url}')
url = property(_get_url, obsolete_setter(_set_url, 'url'))
- def _get_body(self):
+ def _get_body(self) -> bytes:
return self._body
- def _set_body(self, body):
- if body is None:
- self._body = b''
- else:
- self._body = to_bytes(body, self.encoding)
+ def _set_body(self, body: Optional[Union[str, bytes]]) -> None:
+ self._body = b"" if body is None else to_bytes(body, self.encoding)
body = property(_get_body, obsolete_setter(_set_body, 'body'))
@property
- def encoding(self):
+ def encoding(self) -> str:
return self._encoding
- def __str__(self):
- return "<%s %s>" % (self.method, self.url)
+ def __str__(self) -> str:
+ return f"<{self.method} {self.url}>"
__repr__ = __str__
- def copy(self):
- """Return a copy of this Request"""
+ def copy(self) -> "Request":
return self.replace()
- def replace(self, *args, **kwargs):
- """Create a new Request with the same attributes except for those
- given new values.
- """
- for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
- 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
+ def replace(self, *args, **kwargs) -> "Request":
+ """Create a new Request with the same attributes except for those given new values"""
+ for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
@classmethod
- def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+ def from_curl(
+ cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs
+ ) -> RequestTypeVar:
"""Create a Request object from a string containing a `cURL
`_ command. It populates the HTTP method, the
URL, the headers, the cookies and the body. It accepts the same
@@ -132,8 +166,43 @@ class Request(object_ref):
To translate a cURL command into a Scrapy request,
you may use `curl2scrapy `_.
-
- """
+ """
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
request_kwargs.update(kwargs)
return cls(**request_kwargs)
+
+ def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
+ """Return a dictionary containing the Request's data.
+
+ Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
+
+ If a spider is given, this method will try to find out the name of the spider methods used as callback
+ and errback and include them in the output dict, raising an exception if they cannot be found.
+ """
+ d = {
+ "url": self.url, # urls are safe (safe_string_url)
+ "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
+ "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
+ "headers": dict(self.headers),
+ }
+ for attr in self.attributes:
+ d.setdefault(attr, getattr(self, attr))
+ if type(self) is not Request:
+ d["_class"] = self.__module__ + '.' + self.__class__.__name__
+ return d
+
+
+def _find_method(obj, func):
+ """Helper function for Request.to_dict"""
+ # Only instance methods contain ``__func__``
+ if obj and hasattr(func, '__func__'):
+ members = inspect.getmembers(obj, predicate=inspect.ismethod)
+ for name, obj_func in members:
+ # We need to use __func__ to access the original function object because instance
+ # method objects are generated each time attribute is retrieved from instance.
+ #
+ # Reference: The standard type hierarchy
+ # https://docs.python.org/3/reference/datamodel.html
+ if obj_func.__func__ is func.__func__:
+ return name
+ raise ValueError(f"Function {func} is not an instance method in: {obj}")
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index cd4e3373f..c3e112041 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,26 +5,32 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
+from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
from urllib.parse import urljoin, urlencode
-import lxml.html
+from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
from parsel.selector import create_root_node
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
+from scrapy.http.response.text import TextResponse
from scrapy.utils.python import to_bytes, is_listlike
from scrapy.utils.response import get_base_url
+FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
+
+FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
+
+
class FormRequest(Request):
valid_form_methods = ['GET', 'POST']
- def __init__(self, *args, **kwargs):
- formdata = kwargs.pop('formdata', None)
+ def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
if formdata and kwargs.get('method') is None:
kwargs['method'] = 'POST'
- super(FormRequest, self).__init__(*args, **kwargs)
+ super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata
@@ -36,9 +42,19 @@ class FormRequest(Request):
self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
@classmethod
- def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
- clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
-
+ def from_response(
+ cls: Type[FormRequestTypeVar],
+ response: TextResponse,
+ formname: Optional[str] = None,
+ formid: Optional[str] = None,
+ formnumber: Optional[int] = 0,
+ formdata: FormdataType = None,
+ clickdata: Optional[dict] = None,
+ dont_click: bool = False,
+ formxpath: Optional[str] = None,
+ formcss: Optional[str] = None,
+ **kwargs,
+ ) -> FormRequestTypeVar:
kwargs.setdefault('encoding', response.encoding)
if formcss is not None:
@@ -46,7 +62,7 @@ class FormRequest(Request):
formxpath = HTMLTranslator().css_to_xpath(formcss)
form = _get_form(response, formname, formid, formnumber, formxpath)
- formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
+ formdata = _get_inputs(form, formdata, dont_click, clickdata)
url = _get_form_url(form, kwargs.pop('url', None))
method = kwargs.pop('method', form.method)
@@ -58,7 +74,7 @@ class FormRequest(Request):
return cls(url=url, method=method, formdata=formdata, **kwargs)
-def _get_form_url(form, url):
+def _get_form_url(form: FormElement, url: Optional[str]) -> str:
if url is None:
action = form.get('action')
if action is None:
@@ -67,28 +83,33 @@ def _get_form_url(form, url):
return urljoin(form.base_url, url)
-def _urlencode(seq, enc):
+def _urlencode(seq: Iterable, enc: str) -> str:
values = [(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
for v in (vs if is_listlike(vs) else [vs])]
- return urlencode(values, doseq=1)
+ return urlencode(values, doseq=True)
-def _get_form(response, formname, formid, formnumber, formxpath):
- """Find the form element """
- root = create_root_node(response.text, lxml.html.HTMLParser,
- base_url=get_base_url(response))
+def _get_form(
+ response: TextResponse,
+ formname: Optional[str],
+ formid: Optional[str],
+ formnumber: Optional[int],
+ formxpath: Optional[str],
+) -> FormElement:
+ """Find the wanted form element within the given response."""
+ root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
- raise ValueError("No