RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into nameless-spiders

This commit is contained in:
Adrián Chaves 2020-09-01 12:36:19 +02:00 committed by GitHub
parent 437c560e9e 307e35c664
commit f0189b37cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
224 changed files with 4379 additions and 2137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.2.0
current_version = 2.3.0
commit = True
tag = True
tag_name = {new_version}

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
tests/sample_data/** binary

14
.travis.yml
View File

@ -19,16 +19,10 @@ matrix:
python: 3.8
- env: TOXENV=pinned
python: 3.5.2
python: 3.6.1
- env: TOXENV=asyncio-pinned
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
- env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
- env: TOXENV=py
python: 3.5
- env: TOXENV=asyncio
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
python: 3.6.1
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
- env: TOXENV=py
python: 3.6
@ -50,7 +44,7 @@ install:
- |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"

2
README.rst
View File

@ -40,7 +40,7 @@ including a list of features.
Requirements
============
* Python 3.5.2+
* Python 3.6+
* Works on Linux, Windows, macOS, BSD
Install

25
appveyor.yml
View File

@ -1,25 +0,0 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

4
azure-pipelines.yml
View File

@ -4,11 +4,9 @@ pool:
vmImage: 'windows-latest'
strategy:
matrix:
Python35:
python.version: '3.5'
TOXENV: windows-pinned
Python36:
python.version: '3.6'
TOXENV: windows-pinned
Python37:
python.version: '3.7'
Python38:

2
conftest.py
View File

@ -14,8 +14,6 @@ collect_ignore = [
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):

2
docs/_ext/scrapydocs.py
View File

@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
# index entries for setting directives look like:
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
return False

2
docs/conf.py
View File

@ -49,7 +49,7 @@ master_doc = 'index'
# General information about the project.
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the

16
docs/contributing.rst
View File

@ -108,6 +108,11 @@ Well-written patches should:
tox -e docs-coverage
* if you are removing deprecated code, first make sure that at least 1 year
(12 months) has passed since the release that introduced the deprecation.
See :ref:`deprecation-policy`.
.. _submitting-patches:
Submitting patches
@ -194,6 +199,17 @@ In any case, if something is covered in a docstring, use the
documentation instead of duplicating the docstring in files within the
``docs/`` directory.
Documentation updates that cover new or modified features must use Sphinx’s
:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use
``VERSION`` as version, we will replace it with the actual version right before
the corresponding release. When we release a new major or minor version of
Scrapy, we remove these directives if they are older than 3 years.
Documentation about deprecated features must be removed as those features are
deprecated, so that new readers do not run into it. New deprecations and
deprecation removals are documented in the :ref:`release notes <news>`.
Tests
=====

6
docs/faq.rst
View File

@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
To dump into a JSON file::
scrapy crawl myspider -o items.json
scrapy crawl myspider -O items.json
To dump into a CSV file::
scrapy crawl myspider -o items.csv
scrapy crawl myspider -O items.csv
To dump into a XML file::
scrapy crawl myspider -o items.xml
scrapy crawl myspider -O items.xml
For more information see :ref:`topics-feed-exports`

4
docs/intro/install.rst
View File

@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
the PyPy 5.9+ implementation (see :ref:`python:implementations`).
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
Installing Scrapy

26
docs/intro/overview.rst
View File

@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
if next_page is not None:
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.json
scrapy runspider quotes_spider.py -o quotes.jl
When this finishes you will have in the ``quotes.jl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
When this finishes you will have in the ``quotes.json`` file a list of the
quotes in JSON format, containing text and author, looking like this (reformatted
here for better readability)::
[{
"author": "Jane Austen",
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
},
{
"author": "Groucho Marx",
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
},
{
"author": "Steve Martin",
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
},
...]
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
{"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
...
What just happened?

21
docs/intro/tutorial.rst
View File

@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
self.log('Saved file %s' % filename)
self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
@ -190,7 +190,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
@ -405,8 +405,6 @@ to get all of them:
from sys import version_info
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
@ -464,16 +462,15 @@ Storing the scraped data
The simplest way to store the scraped data is by using :ref:`Feed exports
<topics-feed-exports>`, with the following command::
scrapy crawl quotes -o quotes.json
scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also use other formats, like `JSON Lines`_::
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@ -704,7 +701,7 @@ Using spider arguments
You can provide command line arguments to your spiders by using the ``-a``
option when running them::
scrapy crawl quotes -o quotes-humor.json -a tag=humor
scrapy crawl quotes -O quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.

133
docs/news.rst
View File

@ -3,6 +3,139 @@
Release notes
=============
.. _release-2.3.0:
Scrapy 2.3.0 (2020-08-04)
-------------------------
Highlights:
* :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` as a storage backend
* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
output items in batches of up to the specified number of items.
It also serves as a workaround for :ref:`delayed file delivery
<delayed-file-delivery>`, which causes Scrapy to only start item delivery
after the crawl has finished when using certain storage backends
(:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
and now :ref:`GCS <topics-feed-storage-gcs>`).
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
allowing usage from outside Scrapy and a separate release schedule
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* Removed the following classes and their parent modules from
``scrapy.linkextractors``:
* ``htmlparser.HtmlParserLinkExtractor``
* ``regex.RegexLinkExtractor``
* ``sgml.BaseSgmlLinkExtractor``
* ``sgml.SgmlLinkExtractor``
Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead (:issue:`4356`, :issue:`4679`)
Deprecations
~~~~~~~~~~~~
* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
(:issue:`4683`)
New features
~~~~~~~~~~~~
* :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
(:issue:`4250`, :issue:`4434`)
* The :command:`parse` command now allows specifying an output file
(:issue:`4317`, :issue:`4377`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
``--data-raw`` (:issue:`4612`)
* A ``parse`` callback may now be used in built-in spider subclasses, such
as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
:issue:`781`, :issue:`4254` )
Bug fixes
~~~~~~~~~
* Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
:ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
<attrs-items>` (:issue:`4667`, :issue:`4668`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
method to ``POST`` when a request body is specified and no request method
is specified (:issue:`4612`)
* The processing of ANSI escape sequences in enabled in Windows 10.0.14393
and later, where it is required for colored output (:issue:`4393`,
:issue:`4403`)
Documentation
~~~~~~~~~~~~~
* Updated the `OpenSSL cipher list format`_ link in the documentation about
the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
* Simplified the code example in :ref:`topics-loaders-dataclass`
(:issue:`4652`)
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
Quality assurance
~~~~~~~~~~~~~~~~~
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
:issue:`4516`)
* Fixed a silenced error in some scheduler tests (:issue:`4644`,
:issue:`4645`)
* Renewed the localhost certificate used for SSL tests (:issue:`4650`)
* Removed cookie-handling code specific to Python 2 (:issue:`4682`)
* Stopped using Python 2 unicode literal syntax (:issue:`4704`)
* Stopped using a backlash for line continuation (:issue:`4673`)
* Removed unneeded entries from the MyPy exception list (:issue:`4690`)
* Automated tests now pass on Windows as part of our continuous integration
system (:issue:`4458`)
* Automated tests now pass on the latest PyPy version for supported Python
versions in our continuous integration system (:issue:`4504`)
.. _release-2.2.1:
Scrapy 2.2.1 (2020-07-17)
-------------------------
* The :command:`startproject` command no longer makes unintended changes to
the permissions of files in the destination folder, such as removing
execution permissions (:issue:`4662`, :issue:`4666`)
.. _release-2.2.0:
Scrapy 2.2.0 (2020-06-24)

2
docs/topics/api.rst
View File

@ -4,8 +4,6 @@
Core API
========
.. versionadded:: 0.15
This section documents the Scrapy core API, and it's intended for developers of
extensions and middlewares.

12
docs/topics/asyncio.rst
View File

@ -26,3 +26,15 @@ reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.

2
docs/topics/autothrottle.rst
View File

@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies.
AUTOTHROTTLE_TARGET_CONCURRENCY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.1
Default: ``1.0``
Average number of requests Scrapy should be sending in parallel to remote

2
docs/topics/benchmarking.rst
View File

@ -4,8 +4,6 @@
Benchmarking
============
.. versionadded:: 0.17
Scrapy comes with a simple benchmarking suite that spawns a local HTTP server
and crawls it at the maximum possible speed. The goal of this benchmarking is
to get an idea of how Scrapy performs in your hardware, in order to have a

6
docs/topics/commands.rst
View File

@ -6,8 +6,6 @@
Command line tool
=================
.. versionadded:: 0.10
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
just call "commands" or "Scrapy commands".
@ -497,6 +495,8 @@ Supported options:
* ``--output`` or ``-o``: dump scraped items to a file
.. versionadded:: 2.3
.. skip: start
Usage example::
@ -568,8 +568,6 @@ and Platform info, which is useful for bug reports.
bench
-----
.. versionadded:: 0.17
* Syntax: ``scrapy bench``
* Requires project: *no*

4
docs/topics/contracts.rst
View File

@ -4,8 +4,6 @@
Spiders Contracts
=================
.. versionadded:: 0.15
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@ -81,7 +79,7 @@ override three methods:
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function
:type method: collections.abc.Callable
:param args: list of arguments passed into the docstring (whitespace
separated)

15
docs/topics/coroutines.rst
View File

@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
The following are known caveats of the current implementation that we aim
to address in future versions of Scrapy:
- The callback output is not processed until the whole callback finishes.
.. note:: The callback output is not processed until the whole callback
finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
- Because `asynchronous generators were introduced in Python 3.6`_, you
can only use ``yield`` if you are using Python 3.6 or later.
If you need to output multiple items or requests and you are using
Python 3.5, return an iterable (e.g. a list) instead.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
Usage
=====

144
docs/topics/developer-tools.rst
View File

@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping
=================================================
Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with
to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser.
guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_.
@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things:
Inspecting a website
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click
Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this:
@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote:
.. code-block:: html
@ -121,7 +121,7 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
@ -130,45 +130,45 @@ the `has-class-extension`_:
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line.
we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the
The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate
a use case:
a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites.
can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool:
The Network-tool
================
While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_.
take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell:
.. skip: next
@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell:
(...)
>>> view(response)
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
.. image:: _images/network_01.png
:width: 777
@ -189,21 +189,21 @@ bar with the word ``Loading...``.
:alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``.
than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six
new requests.
new requests.
.. image:: _images/network_02.png
:width: 777
@ -212,31 +212,31 @@ new requests.
Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out
where our quotes are coming from:
where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png
:width: 777
:height: 375
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
import scrapy
@ -255,17 +255,17 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading
If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
into our ``url``.
.. _requests-from-curl:
@ -289,14 +289,16 @@ request::
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments:
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.

36
docs/topics/downloader-middleware.rst
View File

@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware:
Multiple cookie sessions per spider
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 0.15
There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
@ -475,8 +473,6 @@ DBM storage backend
.. class:: DbmCacheStorage
.. versionadded:: 0.13
A DBM_ storage backend is also available for the HTTP cache middleware.
By default, it uses the :mod:`dbm`, but you can change it with the
@ -549,15 +545,10 @@ settings:
HTTPCACHE_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.11
Default: ``False``
Whether the HTTP cache will be enabled.
.. versionchanged:: 0.11
Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
.. setting:: HTTPCACHE_EXPIRATION_SECS
HTTPCACHE_EXPIRATION_SECS
@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds.
Cached requests older than this time will be re-downloaded. If zero, cached
requests will never expire.
.. versionchanged:: 0.11
Before 0.11, zero meant cached requests always expire.
.. setting:: HTTPCACHE_DIR
HTTPCACHE_DIR
@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`.
HTTPCACHE_IGNORE_HTTP_CODES
^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``[]``
Don't cache response with these HTTP codes.
@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade
HTTPCACHE_IGNORE_SCHEMES
^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``['file']``
Don't cache responses with these URI schemes.
@ -629,8 +613,6 @@ The class which implements the cache storage backend.
HTTPCACHE_DBM_MODULE
^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend
HTTPCACHE_POLICY
^^^^^^^^^^^^^^^^
.. versionadded:: 0.18
Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
The class which implements the cache policy.
@ -652,8 +632,6 @@ The class which implements the cache policy.
HTTPCACHE_GZIP
^^^^^^^^^^^^^^
.. versionadded:: 1.0
Default: ``False``
If enabled, will compress all cached data with gzip.
@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend.
HTTPCACHE_ALWAYS_STORE
^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``False``
If enabled, will cache pages unconditionally.
@ -684,8 +660,6 @@ responses you feed to the cache middleware.
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``[]``
List of Cache-Control directives in responses to be ignored.
@ -735,8 +709,6 @@ HttpProxyMiddleware
.. module:: scrapy.downloadermiddlewares.httpproxy
:synopsis: Http Proxy Middleware
.. versionadded:: 0.8
.. reqmeta:: proxy
.. class:: HttpProxyMiddleware
@ -817,8 +789,6 @@ RedirectMiddleware settings
REDIRECT_ENABLED
^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Redirect middleware will be enabled.
@ -860,8 +830,6 @@ MetaRefreshMiddleware settings
METAREFRESH_ENABLED
^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.17
Default: ``True``
Whether the Meta Refresh middleware will be enabled.
@ -924,8 +892,6 @@ RetryMiddleware Settings
RETRY_ENABLED
^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Retry middleware will be enabled.
@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings
AJAXCRAWL_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.21
Default: ``False``
Whether the AjaxCrawlMiddleware will be enabled. You may want to

14
docs/topics/email.rst
View File

@ -62,10 +62,10 @@ rest of the framework.
:type smtpport: int
:param smtptls: enforce using SMTP STARTTLS
:type smtptls: boolean
:type smtptls: bool
:param smtpssl: enforce using a secure SSL connection
:type smtpssl: boolean
:type smtpssl: bool
.. classmethod:: from_settings(settings)
@ -79,14 +79,14 @@ rest of the framework.
Send email to the given recipients.
:param to: the e-mail recipients
:type to: str or list of str
:param to: the e-mail recipients as a string or as a list of strings
:type to: str or list
:param subject: the subject of the e-mail
:type subject: str
:param cc: the e-mails to CC
:type cc: str or list of str
:param cc: the e-mails to CC as a string or as a list of strings
:type cc: str or list
:param body: the e-mail body
:type body: str
@ -96,7 +96,7 @@ rest of the framework.
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
attachment and ``file_object`` is a readable file object with the
contents of the attachment
:type attachs: iterable
:type attachs: collections.abc.Iterable
:param mimetype: the MIME type of the e-mail
:type mimetype: str

23
docs/topics/exporters.rst
View File

@ -57,7 +57,7 @@ value of one of their fields::
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
f = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(f)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
@ -98,7 +98,7 @@ Example::
import scrapy
def serialize_price(value):
return '$ %s' % str(value)
return f'$ {str(value)}'
class Product(scrapy.Item):
name = scrapy.Field()
@ -122,7 +122,7 @@ Example::
def serialize_field(self, field, name, value):
if field == 'price':
return '$ %s' % str(value)
return f'$ {str(value)}'
return super(Product, self).serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -166,8 +166,7 @@ BaseItemExporter
By default, this method looks for a serializer :ref:`declared in the item
field <topics-exporters-serializers>` and returns the result of applying
that serializer to the value. If no serializer is found, it returns the
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
value unchanged.
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
@ -217,10 +216,7 @@ BaseItemExporter
.. attribute:: encoding
The encoding that will be used to encode unicode values. This only
affects unicode values (which are always serialized to str using this
encoding). Other value types are passed unchanged to the specific
serialization library.
The output character encoding.
.. attribute:: indent
@ -296,7 +292,7 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
@ -309,12 +305,17 @@ CsvItemExporter
:param include_headers_line: If enabled, makes the exporter output a header
line with the field names taken from
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
:type include_headers_line: boolean
:type include_headers_line: bool
:param join_multivalued: The char (or chars) that will be used for joining
multi-valued fields, if found.
:type include_headers_line: str
:param errors: The optional string that specifies how encoding and decoding
errors are to be handled. For more information see
:class:`io.TextIOWrapper`.
:type errors: str
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
:func:`csv.writer` function, so you can use any :func:`csv.writer` function

4
docs/topics/extensions.rst
View File

@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items.
CLOSESPIDER_PAGECOUNT
"""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of responses to crawl. If the spider
@ -302,8 +300,6 @@ number of crawled responses.
CLOSESPIDER_ERRORCOUNT
""""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of errors to receive before

187
docs/topics/feed-exports.rst
View File

@ -4,8 +4,6 @@
Feed exports
============
.. versionadded:: 0.10
One of the most frequently required features when implementing scrapers is
being able to store the scraped data properly and, quite often, that means
generating an "export file" with the scraped data (commonly called "export
@ -100,6 +98,7 @@ The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
* :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
@ -169,6 +168,9 @@ FTP supports two different connection modes: `active or passive
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-s3:
S3
@ -194,11 +196,16 @@ You can also define a custom ACL for exported feeds using this setting:
* :setting:`FEED_STORAGE_S3_ACL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-gcs:
Google Cloud Storage (GCS)
--------------------------
.. versionadded:: 2.3
The feeds are stored on `Google Cloud Storage`_.
* URI scheme: ``gs``
@ -206,7 +213,7 @@ The feeds are stored on `Google Cloud Storage`_.
* ``gs://mybucket/path/to/export.csv``
* Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
* Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
@ -215,6 +222,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
* :setting:`FEED_STORAGE_GCS_ACL`
* :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
.. _topics-feed-storage-stdout:
Standard output
@ -227,6 +239,26 @@ The feeds are written to the standard output of the Scrapy process.
* Required external libraries: none
.. _delayed-file-delivery:
Delayed file delivery
---------------------
As indicated above, some of the described storage backends use delayed file
delivery.
These storage backends do not upload items to the feed URI as those items are
scraped. Instead, Scrapy writes items into a temporary local file, and only
once all the file contents have been written (i.e. at the end of the crawl) is
that file uploaded to the feed URI.
If you want item delivery to start earlier when using one of these storage
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
in multiple files, with the specified maximum item count per file. That way, as
soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
Settings
========
@ -241,6 +273,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
@ -256,6 +289,7 @@ Default: ``{}``
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
This setting is required for enabling the feed export feature.
See :ref:`topics-feed-storage-backends` for supported URI schemes.
@ -283,15 +317,43 @@ For instance::
}
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition.
as a fallback value if that key is not provided for a specific feed definition:
- ``format``: the :ref:`serialization format <topics-feed-format>`.
This setting is mandatory, there is no fallback value.
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
The default value depends on the :ref:`storage backend
<topics-feed-storage-backends>`:
- :ref:`topics-feed-storage-fs`: ``False``
- :ref:`topics-feed-storage-ftp`: ``True``
.. note:: Some FTP servers may not support appending to files (the
``APPE`` FTP command).
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
* ``format``: the serialization format to be used for the feed.
See :ref:`topics-feed-format` for possible values.
Mandatory, no fallback setting
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
.. setting:: FEED_EXPORT_ENCODING
@ -446,6 +508,109 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
-----------------------------
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
placeholders in the feed URI to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
alter the number format. For example, to make the batch ID a 5-digit
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
FEED_EXPORT_BATCH_ITEM_COUNT = 100
And your :command:`crawl` command line is::
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
The command line above can generate a directory tree like::
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fewer.
.. setting:: FEED_URI_PARAMS
FEED_URI_PARAMS
---------------
Default: ``None``
A string with the import path of a function to set the parameters to apply with
:ref:`printf-style string formatting <python:old-string-formatting>` to the
feed URI.
The function signature should be as follows:
.. function:: uri_params(params, spider)
Return a :class:`dict` of key-value pairs to apply to the feed URI using
:ref:`printf-style string formatting <python:old-string-formatting>`.
:param params: default key-value pairs
Specifically:
- ``batch_id``: ID of the file batch. See
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.spiders.Spider
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore

6
docs/topics/item-pipeline.rst
View File

@ -96,7 +96,7 @@ contain a price::
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
@ -211,7 +211,7 @@ item.
# Save screenshot to file, filename will be hash of url.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
@ -240,7 +240,7 @@ returns multiples items with the same id::
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item)
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
return item

4
docs/topics/leaks.rst
View File

@ -102,7 +102,7 @@ A real example
Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
:param ignore: if given, all objects from the specified class (or tuple of
classes) will be ignored.
:type ignore: class or classes tuple
:type ignore: type or tuple
.. function:: get_oldest(class_name)

14
docs/topics/link-extractors.rst
View File

@ -46,13 +46,13 @@ LxmlLinkExtractor
:param allow: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be extracted. If not
given (or empty), it will match all links.
:type allow: a regular expression (or list of)
:type allow: str or list
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
:type deny: str or list
:param allow_domains: a single value or a list of string containing
domains which will be considered for extracting the links
@ -88,7 +88,7 @@ LxmlLinkExtractor
that the link's text must match in order to be extracted. If not
given (or empty), it will match all links. If a list of regular expressions is
given, the link will be extracted if it matches at least one.
:type restrict_text: a regular expression (or list of)
:type restrict_text: str or list
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
@ -106,11 +106,11 @@ LxmlLinkExtractor
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:type canonicalize: bool
:param unique: whether duplicate filtering should be applied to extracted
links.
:type unique: boolean
:type unique: bool
:param process_value: a function which receives each value extracted from
the tag and attributes scanned and can modify the value and return a
@ -132,7 +132,7 @@ LxmlLinkExtractor
if m:
return m.group(1)
:type process_value: callable
:type process_value: collections.abc.Callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
@ -141,7 +141,7 @@ LxmlLinkExtractor
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
:type strip: bool
.. automethod:: extract_links

10
docs/topics/loaders.rst
View File

@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example::
default_output_processor = TakeFirst()
name_in = MapCompose(unicode.title)
name_in = MapCompose(str.title)
name_out = Join()
price_in = MapCompose(unicode.strip)
price_in = MapCompose(str.strip)
# ...
@ -237,10 +237,10 @@ metadata. Here is an example::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:

24
docs/topics/media-pipeline.rst
View File

@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.

90
docs/topics/request-response.rst
View File

@ -33,7 +33,7 @@ Request objects
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: string
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
@ -42,21 +42,21 @@ Request objects
:meth:`~scrapy.spiders.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
:type callback: callable
:type callback: collections.abc.Callable
:param method: the HTTP method of this request. Defaults to ``'GET'``.
:type method: string
:type method: str
:param meta: the initial values for the :attr:`Request.meta` attribute. If
given, the dict passed in this parameter will be shallow copied.
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
:type body: str or unicode
:param body: the request body. If a string is passed, then it's encoded as
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty bytes object is stored. Regardless of the
type of this argument, the final value stored will be a bytes object
(never a string or ``None``).
:type body: bytes or str
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
@ -106,8 +106,8 @@ Request objects
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to ``str`` (if given as ``unicode``).
:type encoding: string
body to bytes (if given as a string).
:type encoding: str
:param priority: the priority of this request (defaults to ``0``).
The priority is used by the scheduler to define the order used to process
@ -119,7 +119,7 @@ Request objects
the scheduler. This is used when you want to perform an identical
request multiple times, to ignore the duplicates filter. Use it with
care, or you will get into crawling loops. Default to ``False``.
:type dont_filter: boolean
:type dont_filter: bool
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
@ -131,7 +131,7 @@ Request objects
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: callable
:type errback: collections.abc.Callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
@ -159,7 +159,7 @@ Request objects
.. attribute:: Request.body
A str that contains the request body.
The request body as bytes.
This attribute is read-only. To change the body of a Request use
:meth:`replace`.
@ -485,7 +485,7 @@ fields with form data from :class:`Response` objects.
:param formdata: is a dictionary (or iterable of (key, value) tuples)
containing HTML Form data which will be url-encoded and assigned to the
body of the request.
:type formdata: dict or iterable of tuples
:type formdata: dict or collections.abc.Iterable
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
@ -517,20 +517,20 @@ fields with form data from :class:`Response` objects.
:type response: :class:`Response` object
:param formname: if given, the form with name attribute set to this value will be used.
:type formname: string
:type formname: str
:param formid: if given, the form with id attribute set to this value will be used.
:type formid: string
:type formid: str
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
:type formxpath: str
:param formcss: if given, the first form that matches the css selector will be used.
:type formcss: string
:type formcss: str
:param formnumber: the number of form to use, when the response contains
multiple forms. The first one (and also the default) is ``0``.
:type formnumber: integer
:type formnumber: int
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
@ -548,23 +548,11 @@ fields with form data from :class:`Response` objects.
:param dont_click: If True, the form data will be submitted without
clicking in any element.
:type dont_click: boolean
:type dont_click: bool
The other parameters of this class method are passed directly to the
:class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
.. versionadded:: 0.17
The ``formxpath`` parameter.
.. versionadded:: 1.1.0
The ``formcss`` parameter.
.. versionadded:: 1.1.0
The ``formid`` parameter.
Request usage examples
----------------------
@ -636,7 +624,7 @@ dealing with JSON requests.
if :attr:`Request.body` argument is provided this parameter will be ignored.
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
:type data: object
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
@ -663,16 +651,16 @@ Response objects
downloaded (by the Downloader) and fed to the Spiders for processing.
:param url: the URL of this response
:type url: string
:type url: str
:param status: the HTTP status of the response. Defaults to ``200``.
:type status: integer
:type status: int
:param headers: the headers of this response. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. To access the decoded text as str you can use
:param body: the response body. To access the decoded text as a string, use
``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
@ -720,10 +708,10 @@ Response objects
.. attribute:: Response.body
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the unicode version use
:attr:`TextResponse.text` (only available in :class:`TextResponse`
and subclasses).
The response body as bytes.
If you want the body as a string, use :attr:`TextResponse.text` (only
available in :class:`TextResponse` and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@ -842,18 +830,18 @@ TextResponse objects
is the same as for the :class:`Response` class and is not documented here.
:param encoding: is a string which contains the encoding to use for this
response. If you create a :class:`TextResponse` object with a unicode
body, it will be encoded using this encoding (remember the body attribute
is always a string). If ``encoding`` is ``None`` (default value), the
encoding will be looked up in the response headers and body instead.
:type encoding: string
response. If you create a :class:`TextResponse` object with a string as
body, it will be converted to bytes encoded using this encoding. If
*encoding* is ``None`` (default), the encoding will be looked up in the
response headers and body instead.
:type encoding: str
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:
.. attribute:: TextResponse.text
Response body, as unicode.
Response body, as a string.
The same as ``response.body.decode(response.encoding)``, but the
result is cached after the first call, so you can access
@ -861,9 +849,11 @@ TextResponse objects
.. note::
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically ``ascii``) instead of the response encoding.
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. attribute:: TextResponse.encoding

18
docs/topics/selectors.rst
View File

@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
markup as an unicode string (in ``text`` argument).
markup as a string (in ``text`` argument).
Usually there is no need to construct Scrapy selectors manually:
``response`` object is available in Spider callbacks, so in most cases
it is more convenient to use ``response.css()`` and ``response.xpath()``
@ -327,8 +328,9 @@ too. Here's an example:
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
... href_xpath = link.xpath('@href').get()
... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
@ -383,7 +385,7 @@ Using selectors with regular expressions
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
data using regular expressions. However, unlike using ``.xpath()`` or
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
``.css()`` methods, ``.re()`` returns a list of strings. So you
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
@ -734,7 +736,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = u"""
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
@ -765,7 +767,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
>>> doc = u"""
>>> doc = """
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@ -821,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
... print(" properties: %s" % (props.getall()))
... print(f" properties: {props.getall()}")
... print("")
current scope: ['http://schema.org/Product']
@ -989,7 +991,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel.xpath("//h1")
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
returning a list of strings::
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag

52
docs/topics/settings.rst
View File

@ -98,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Import paths and classes
========================
.. versionadded:: VERSION
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
- As a string containing the import path of that object
- As the object itself
For example::
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
}
.. note:: Passing non-callable objects is not supported.
How to access settings
======================
@ -110,7 +136,7 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com']
def parse(self, response):
print("Existing settings: %s" % self.settings.attributes.keys())
print(f"Existing settings: {self.settings.attributes.keys()}")
.. note::
The ``settings`` attribute is set in the base Spider class after the spider
@ -216,6 +242,26 @@ Default: ``None``
The name of the region associated with the AWS client.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
------------------
Default: ``None``
Import path of a given asyncio event loop class.
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
event loop will be used.
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
.. setting:: BOT_NAME
BOT_NAME
@ -1030,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`.
MEMUSAGE_CHECK_INTERVAL_SECONDS
-------------------------------
.. versionadded:: 1.1
Default: ``60.0``
Scope: ``scrapy.extensions.memusage``
@ -1336,8 +1380,6 @@ as a name.
SPIDER_LOADER_WARN_ONLY
-----------------------
.. versionadded:: 1.3.3
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,

5
docs/topics/signals.rst
View File

@ -423,6 +423,11 @@ response_received
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: The ``request`` argument might not contain the original request that
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
the :class:`~scrapy.http.Response` object and sets a specific ``request``
attribute.
response_downloaded
~~~~~~~~~~~~~~~~~~~

6
docs/topics/spider-middleware.rst
View File

@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
.. method:: process_start_requests(start_requests, spider)
.. versionadded:: 0.15
This method is called with the start requests of the spider, and works
similarly to the :meth:`process_spider_output` method, except that it
doesn't have a response associated and must return only requests (not
@ -341,8 +339,6 @@ RefererMiddleware settings
REFERER_ENABLED
^^^^^^^^^^^^^^^
.. versionadded:: 0.15
Default: ``True``
Whether to enable referer middleware.
@ -352,8 +348,6 @@ Whether to enable referer middleware.
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy

4
docs/topics/spiders.rst
View File

@ -282,7 +282,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = ['http://www.example.com/categories/%s' % category]
self.start_urls = [f'http://www.example.com/categories/{category}']
# ...
The default `__init__` method will take any spider arguments
@ -295,7 +295,7 @@ The above example can also be written as follows::
name = 'myspider'
def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.

2
docs/utils/linkfix.py
View File

@ -23,7 +23,7 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:

22
docs/versioning.rst
View File

@ -1,7 +1,7 @@
.. _versioning:
============================
Versioning and API Stability
Versioning and API stability
============================
Versioning
@ -34,7 +34,7 @@ For example:
production)
API Stability
API stability
=============
API stability was one of the major goals for the *1.0* release.
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
same way.
.. _deprecation-policy:
Deprecation policy
==================
We aim to maintain support for deprecated Scrapy features for at least 1 year.
For example, if a feature is deprecated in a Scrapy version released on
June 15th 2020, that feature should continue to work in versions released on
June 14th 2021 or before that.
Any new Scrapy release after a year *may* remove support for that deprecated
feature.
All deprecated features removed in a Scrapy release are explicitly mentioned in
the :ref:`release notes <news>`.
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases

2
extras/qps-bench-server.py
View File

@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args:
latency = float(request.args['latency'][0])

6
extras/qpsclient.py
View File

@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += '?latency={0}'.format(self.latency)
url += f'?latency={self.latency}'
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else:
urls = [url]

2
pylintrc
View File

@ -68,6 +68,7 @@ disable=abstract-method,
pointless-statement,
pointless-string-statement,
protected-access,
raise-missing-from,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
@ -75,6 +76,7 @@ disable=abstract-method,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,

1
pytest.ini
View File

@ -40,3 +40,4 @@ flake8-ignore =
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

2
scrapy/VERSION
View File

@ -1 +1 @@
2.2.0
2.3.0

4
scrapy/__init__.py
View File

@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 5, 2):
print("Scrapy %s requires Python 3.5.2" % __version__)
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
sys.exit(1)

16
scrapy/cmdline.py
View File

@ -42,7 +42,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
else:
raise Exception("Invalid entry point %s" % entry_point.name)
raise Exception(f"Invalid entry point {entry_point.name}")
return cmds
@ -65,11 +65,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
@ -79,7 +79,7 @@ def _print_commands(settings, inproject):
print("Available commands:")
cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()):
print(" %-13s %s" % (cmdname, cmdclass.short_desc()))
print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject:
print()
print(" [ more ] More commands available when run from project directory")
@ -89,7 +89,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
@ -131,7 +131,7 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
@ -153,7 +153,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)

17
scrapy/commands/__init__.py
View File

@ -61,7 +61,7 @@ class ScrapyCommand:
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
if opts.output or opts.overwrite_output:
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

38
scrapy/commands/check.py
View File

@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else ""
writeln(self.separator2)
writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
writeln()
infos = []
@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED")
failed, errored = map(len, (self.failures, self.errors))
if failed:
infos.append("failures=%d" % failed)
infos.append(f"failures={failed}")
if errored:
infos.append("errors=%d" % errored)
infos.append(f"errors={errored}")
else:
write("OK")
if infos:
writeln(" (%s)" % (", ".join(infos),))
writeln(f" ({', '.join(infos)})")
else:
write("\n")
@ -78,19 +78,19 @@ class Command(ScrapyCommand):
elif tested_methods:
self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())

4
scrapy/commands/edit.py
View File

@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err("Spider not found: %s" % args[0])
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
self.exitcode = os.system('%s "%s"' % (editor, sfile))
self.exitcode = os.system(f'{editor} "{sfile}"')

60
scrapy/commands/genspider.py
View File

@ -66,31 +66,25 @@ class Command(ScrapyCommand):
print("Cannot create a spider with the same name as your project")
return
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider already exists and not --force then halt
if not opts.force:
print("Spider %r already exists in module:" % name)
print(" %s" % spidercls.__module__)
return
if not opts.force and self._spider_exists(name):
return
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
if opts.edit:
self.exitcode = os.system('scrapy edit "%s"' % name)
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
'classname': f'{capitalized_module}Spider'
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -98,26 +92,54 @@ class Command(ScrapyCommand):
else:
spiders_module = None
spiders_dir = "."
spider_file = "%s.py" % join(spiders_dir, module)
spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))
print("in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, '%s.tmpl' % template)
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
return template_file
print("Unable to find template: %s\n" % template)
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(" %s" % splitext(filename)[0])
print(f" {splitext(filename)[0]}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
return True
return False
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider with same name exists
print(f"Spider {name!r} already exists in module:")
print(f" {spidercls.__module__}")
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
return True
return False
@property
def templates_dir(self):

4
scrapy/commands/parse.py
View File

@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:

8
scrapy/commands/runspider.py
View File

@ -12,7 +12,7 @@ def _import_file(filepath):
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
raise ValueError("Not a Python source file: %s" % abspath)
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
@ -42,16 +42,16 @@ class Command(BaseRunSpiderCommand):
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
raise UsageError("File not found: %s\n" % filename)
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError("Unable to load %r: %s\n" % (filename, e))
raise UsageError(f"Unable to load {filename!r}: {e}\n")
require_name = self.settings.getbool('SPIDER_LOADER_REQUIRE_NAME')
spclasses = list(iter_spider_classes(module,
require_name=require_name))
if not spclasses:
raise UsageError("No spider found in file: %s\n" % filename)
raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs)

12
scrapy/commands/startproject.py
View File

@ -52,7 +52,7 @@ class Command(ScrapyCommand):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
print(f'Error: Module {project_name!r} already exists')
else:
return True
return False
@ -100,7 +100,7 @@ class Command(ScrapyCommand):
if exists(join(project_dir, 'scrapy.cfg')):
self.exitcode = 1
print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
return
if not self._is_valid_name(project_name):
@ -113,11 +113,11 @@ class Command(ScrapyCommand):
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
print("You can start your first spider with:")
print(" cd %s" % project_dir)
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property

5
scrapy/commands/version.py
View File

@ -23,8 +23,7 @@ class Command(ScrapyCommand):
if opts.verbose:
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)
patt = "%-{}s : %s".format(width)
for name, version in versions:
print(patt % (name, version))
print(f"{name:<{width}} : {version}")
else:
print("Scrapy %s" % scrapy.__version__)
print(f"Scrapy {scrapy.__version__}")

2
scrapy/commands/view.py
View File

@ -11,7 +11,7 @@ class Command(fetch.Command):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
def add_options(self, parser):
super(Command, self).add_options(parser)
super().add_options(parser)
parser.remove_option("--headers")
def _print_response(self, response, opts):

8
scrapy/contracts/__init__.py
View File

@ -112,8 +112,8 @@ class Contract:
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name)
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
@ -172,8 +172,8 @@ def _create_testcase(method, desc):
class ContractTestCase(TestCase):
def __str__(_self):
return "[%s] %s (%s)" % (spider, method.__name__, desc)
return f"[{spider}] {method.__name__} ({desc})"
name = '%s_%s' % (spider, method.__name__)
name = f'{spider}_{method.__name__}'
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

14
scrapy/contracts/default.py
View File

@ -56,12 +56,11 @@ class ReturnsContract(Contract):
}
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
)
self.obj_name = self.args[0] or None
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
@ -88,10 +87,9 @@ class ReturnsContract(Contract):
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
expected = f'{self.min_bound}..{self.max_bound}'
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
class ScrapesContract(Contract):
@ -106,5 +104,5 @@ class ScrapesContract(Contract):
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
missing_str = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str)
missing_fields = ", ".join(missing)
raise ContractFail(f"Missing fields: {missing_fields}")

16
scrapy/core/downloader/__init__.py
View File

@ -41,17 +41,17 @@ class Slot:
def __repr__(self):
cls_name = self.__class__.__name__
return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % (
cls_name, self.concurrency, self.delay, self.randomize_delay)
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
def __str__(self):
return (
"<downloader.Slot concurrency=%r delay=%0.2f randomize_delay=%r "
"len(active)=%d len(queue)=%d len(transferring)=%d lastseen=%s>" % (
self.concurrency, self.delay, self.randomize_delay,
len(self.active), len(self.queue), len(self.transferring),
datetime.fromtimestamp(self.lastseen).isoformat()
)
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
)

4
scrapy/core/downloader/contextfactory.py
View File

@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),

3
scrapy/core/downloader/handlers/__init__.py
View File

@ -71,8 +71,7 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported("Unsupported URL scheme '%s': %s" %
(scheme, self._notconfigured[scheme]))
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
@defer.inlineCallbacks

48
scrapy/core/downloader/handlers/http11.py
View File

@ -60,11 +60,11 @@ class HTTP11DownloadHandler:
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
msg = f"""
'{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \
argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \
`tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them."""
warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
else:
extra = rcvd_bytes[:32]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
self._host, self._port, extra)))
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
)
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@ -178,7 +179,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@ -215,7 +216,7 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -235,7 +236,7 @@ class TunnelingAgent(Agent):
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
return super(TunnelingAgent, self)._requestWithEndpoint(
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
@ -249,7 +250,7 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
super(ScrapyProxyAgent, self).__init__(
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
@ -371,7 +372,7 @@ class ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
@ -394,13 +395,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
error_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
logger.warning(warning_msg, warning_args)
logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
raise defer.CancelledError(error_msg % error_args)
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@ -523,11 +525,11 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()

6
scrapy/core/downloader/handlers/s3.py
View File

@ -56,7 +56,7 @@ class S3DownloadHandler:
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
@ -85,14 +85,14 @@ class S3DownloadHandler:
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
headers=request.headers.to_unicode_dict(),
data=request.body)
self._signer.add_auth(awsrequest)

15
scrapy/core/downloader/middleware.py
View File

@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
f"Middleware {method.__self__.__class__.__name__}"
".process_request must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
return response
@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__self__.__class__.__name__}"
".process_response must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
return response
@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__self__.__class__.__name__}"
".process_exception must return None, Response or "
f"Request, got {type(response)}"
)
if response:
return response

2
scrapy/core/downloader/tls.py
View File

@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
"""
def __init__(self, hostname, ctx, verbose_logging=False):
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
super().__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):

98
scrapy/core/downloader/webclient.py
View File

@ -1,9 +1,9 @@
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet import defer, reactor
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
@ -88,22 +88,38 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds."
% (self.factory.url, self.factory.timeout)))
defer.TimeoutError(f"Getting {self.factory.url} took longer "
f"than {self.factory.timeout} seconds."))
class ScrapyHTTPClientFactory(HTTPClientFactory):
"""Scrapy implementation of the HTTPClientFactory overwriting the
setUrl method to make use of our Url object that cache the parse
result.
"""
# This class used to inherit from Twisted’s
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overriden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
@ -138,21 +154,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
elif self.method == b'POST':
self.headers['Content-Length'] = 0
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
def __repr__(self):
return f"<{self.__class__.__name__}: {self.url}>"
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def _cancelTimeout(self, result, timeoutCall):
if timeoutCall.active():
timeoutCall.cancel()
return result
def buildProtocol(self, addr):
p = ClientFactory.buildProtocol(self, addr)
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
def gotStatus(self, version, status, message):
"""
Set the status of the request on us.
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}
"""
self.version, self.status, self.message = version, status, message
def page(self, page):
if self.waiting:
self.waiting = 0
self.deferred.callback(page)
def noPage(self, reason):
if self.waiting:
self.waiting = 0
self.deferred.errback(reason)
def clientConnectionFailed(self, _, reason):
"""
When a connection attempt fails, the request cannot be issued. If no
result has yet been provided to the result Deferred, provide the
connection failure reason as an error result.
"""
if self.waiting:
self.waiting = 0
# If the connection attempt failed, there is nothing more to
# disconnect, so just fire that Deferred now.
self._disconnectedDeferred.callback(None)
self.deferred.errback(reason)

25
scrapy/core/engine.py
View File

@ -171,8 +171,8 @@ class ExecutionEngine:
def _handle_downloader_output(self, response, request, spider):
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
"Incorrect type: expected Request, Response or Failure, got "
f"{type(response)}: {response!r}"
)
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
@ -214,7 +214,7 @@ class ExecutionEngine:
def crawl(self, request, spider):
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}")
self.schedule(request, spider)
self.slot.nextcall.schedule()
@ -239,16 +239,21 @@ class ExecutionEngine:
def _on_success(response):
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
"Incorrect type: expected Response or Request, got "
f"{type(response)}: {response!r}"
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
if response.request is None:
response.request = request
logkws = self.logformatter.crawled(response.request, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
self.signals.send_catch_log(signals.response_received,
response=response, request=request, spider=spider)
self.signals.send_catch_log(
signal=signals.response_received,
response=response,
request=response.request,
spider=spider,
)
return response
def _on_complete(_):
@ -263,7 +268,7 @@ class ExecutionEngine:
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)

64
scrapy/core/scraper.py
View File

@ -12,7 +12,7 @@ from scrapy import signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
@ -120,40 +120,40 @@ class Scraper:
response, request, deferred = slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
def _scrape(self, response, request, spider):
"""Handle the downloaded response or failure through the spider
callback/errback"""
if not isinstance(response, (Response, Failure)):
raise TypeError(
"Incorrect type: expected Response or Failure, got %s: %r"
% (type(response), response)
)
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
def _scrape(self, result, request, spider):
"""
Handle the downloaded response or failure through the spider callback/errback
"""
if not isinstance(result, (Response, Failure)):
raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
dfd = self._scrape2(result, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
def _scrape2(self, request_result, request, spider):
"""Handle the different cases of request's result been a Response or a
Failure"""
if not isinstance(request_result, Failure):
return self.spidermw.scrape_response(
self.call_spider, request_result, request, spider)
else:
dfd = self.call_spider(request_result, request, spider)
return dfd.addErrback(
self._log_download_errors, request_result, request, spider)
def _scrape2(self, result, request, spider):
"""
Handle the different cases of request's result been a Response or a Failure
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
else: # result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
callback = request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
warn_on_generator_with_return_value(spider, request.errback)
dfd.addCallbacks(callback=callback,
errback=request.errback,
callbackKeywords=request.cb_kwargs)
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallback(callback, **result.request.cb_kwargs)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
@ -173,7 +173,7 @@ class Scraper:
spider=spider
)
self.crawler.stats.inc_value(
"spider_exceptions/%s" % _failure.value.__class__.__name__,
f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=spider
)

22
scrapy/core/spidermw.py
View File

@ -19,10 +19,7 @@ def _isiterable(possible_iterator):
def _fname(f):
return "{}.{}".format(
f.__self__.__class__.__name__,
f.__func__.__name__
)
return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
class SpiderMiddlewareManager(MiddlewareManager):
@ -34,7 +31,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
def _add_middleware(self, mw):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
super()._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
try:
result = method(response=response, spider=spider)
if result is not None:
msg = "Middleware {} must return None or raise an exception, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return None "
f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput:
raise
except Exception:
@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
msg = "Middleware {} must return None or an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return None "
f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure
def process_spider_output(result, start_index=0):
@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = _evaluate_iterable(result, method_index + 1, recovered)
else:
msg = "Middleware {} must return an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
return MutableChain(result, recovered)

6
scrapy/crawler.py
View File

@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
"""
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@ -307,7 +307,7 @@ class CrawlerProcess(CrawlerRunner):
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
from twisted.internet import reactor
@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner):
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"])
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
super()._handle_twisted_reactor()

14
scrapy/downloadermiddlewares/cookies.py
View File

@ -54,8 +54,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
msg = "Sending cookies to: {}\n{}".format(request, cookies)
cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _debug_set_cookie(self, response, spider):
@ -63,8 +63,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
msg = "Received cookies from: {}\n{}".format(response, cookies)
cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie, request):
@ -74,7 +74,7 @@ class CookiesMiddleware:
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
if not cookie.get(key):
if cookie.get(key) is None:
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
@ -90,9 +90,9 @@ class CookiesMiddleware:
request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
for key, value in decoded.items(): # path, domain
cookie_str += "; {}={}".format(key.capitalize(), value)
cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
def _get_request_cookies(self, jar, request):

41
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,4 +1,5 @@
from email.utils import formatdate
from typing import Optional, Type, TypeVar
from twisted.internet import defer
from twisted.internet.error import (
@ -13,10 +14,19 @@ from twisted.internet.error import (
from twisted.web.client import ResponseFailed
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@ -24,7 +34,7 @@ class HttpCacheMiddleware:
ConnectionLost, TCPTimedOutError, ResponseFailed,
IOError)
def __init__(self, settings, stats):
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
@ -33,26 +43,26 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self.storage.open_spider(spider)
def spider_closed(self, spider):
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
def process_request(self, request, spider):
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
if request.meta.get('dont_cache', False):
return
return None
# Skip uncacheable requests
if not self.policy.should_cache_request(request):
request.meta['_dont_cache'] = True # flag as uncacheable
return
return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
@ -61,7 +71,7 @@ class HttpCacheMiddleware:
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
raise IgnoreRequest("Ignored request not in cache: %s" % request)
return # first time request
return None # first time request
# Return cached response only if not expired
cachedresponse.flags.append('cached')
@ -73,7 +83,9 @@ class HttpCacheMiddleware:
# process_response hook
request.meta['cached_response'] = cachedresponse
def process_response(self, request, response, spider):
return None
def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
if request.meta.get('dont_cache', False):
return response
@ -85,7 +97,7 @@ class HttpCacheMiddleware:
# RFC2616 requires origin server to set Date header,
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
if 'Date' not in response.headers:
response.headers['Date'] = formatdate(usegmt=1)
response.headers['Date'] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop('cached_response', None)
@ -102,13 +114,18 @@ class HttpCacheMiddleware:
self._cache_response(spider, response, request, cachedresponse)
return response
def process_exception(self, request, exception, spider):
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Optional[Response]:
cachedresponse = request.meta.pop('cached_response', None)
if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
self.stats.inc_value('httpcache/errorrecovery', spider=spider)
return cachedresponse
return None
def _cache_response(self, spider, response, request, cachedresponse):
def _cache_response(
self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value('httpcache/store', spider=spider)
self.storage.store_response(spider, request, response)

2
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -24,7 +24,7 @@ class HttpProxyMiddleware:
def _basic_auth_header(self, username, password):
user_pass = to_bytes(
'%s:%s' % (unquote(username), unquote(password)),
f'{unquote(username)}:{unquote(password)}',
encoding=self.auth_encoding)
return base64.b64encode(user_pass)

2
scrapy/downloadermiddlewares/redirect.py
View File

@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = 'METAREFRESH_ENABLED'
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
super().__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')

2
scrapy/downloadermiddlewares/retry.py
View File

@ -88,7 +88,7 @@ class RetryMiddleware:
reason = global_object_name(reason.__class__)
stats.inc_value('retry/count')
stats.inc_value('retry/reason_count/%s' % reason)
stats.inc_value(f'retry/reason_count/{reason}')
return retryreq
else:
stats.inc_value('retry/max_reached')

6
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -61,7 +61,7 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers:
self._parsers[netloc] = Deferred()
robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
@ -94,7 +94,7 @@ class RobotsTxtMiddleware:
def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
@ -102,7 +102,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, failure, netloc):
if failure.type is not IgnoreRequest:
key = 'robotstxt/exception_count/{}'.format(failure.type)
key = f'robotstxt/exception_count/{failure.type}'
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = None

6
scrapy/downloadermiddlewares/stats.py
View File

@ -17,13 +17,13 @@ class DownloaderStats:
def process_request(self, request, spider):
self.stats.inc_value('downloader/request_count', spider=spider)
self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
reqlen = len(request_httprepr(request))
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response_httprepr(response))
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response
@ -31,4 +31,4 @@ class DownloaderStats:
def process_exception(self, request, exception, spider):
ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)

4
scrapy/exceptions.py
View File

@ -37,7 +37,7 @@ class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason='cancelled'):
super(CloseSpider, self).__init__()
super().__init__()
self.reason = reason
@ -74,7 +74,7 @@ class UsageError(Exception):
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
super().__init__(*a, **kw)
class ScrapyDeprecationWarning(Warning):

17
scrapy/exporters.py
View File

@ -39,7 +39,7 @@ class BaseItemExporter:
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None)
if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
def export_item(self, item):
raise NotImplementedError
@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
errors=errors,
)
self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True
@ -243,12 +244,8 @@ class CsvItemExporter(BaseItemExporter):
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
if isinstance(item, dict):
# for dicts try using fields of the first item
self.fields_to_export = list(item.keys())
else:
# use fields declared in Item
self.fields_to_export = list(item.fields.keys())
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
row = list(self._build_row(self.fields_to_export))
self.csv_writer.writerow(row)
@ -305,7 +302,7 @@ class PythonItemExporter(BaseItemExporter):
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
super(PythonItemExporter, self)._configure(options, dont_fail)
super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",

2
scrapy/extensions/corestats.py
View File

@ -43,4 +43,4 @@ class CoreStats:
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
self.stats.inc_value('item_dropped_count', spider=spider)
self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)

2
scrapy/extensions/debug.py
View File

@ -48,7 +48,7 @@ class StackTraceDump:
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, '')
dump = ''.join(traceback.format_stack(frame))
dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
dumps += f"# Thread: {name}({id_})\n{dump}\n"
return dumps

295
scrapy/extensions/feedexport.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
import logging
import os
import re
import sys
import warnings
from datetime import datetime
@ -23,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
if 'feed_options' in argument_names:
kwargs['feed_options'] = feed_options
else:
warnings.warn(
"{} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy."
.format(builder.__qualname__),
category=ScrapyDeprecationWarning
)
return builder(*preargs, uri, *args, **kwargs)
class IFeedStorage(Interface):
"""Interface that all Feed Storages must implement"""
def __init__(uri):
"""Initialize the storage with the parameters given in the URI"""
def __init__(uri, *, feed_options=None):
"""Initialize the storage with the parameters given in the URI and the
feed-specific options (see :setting:`FEEDS`)"""
def open(spider):
"""Open the storage for the given spider. It must return a file-like
@ -63,10 +81,15 @@ class BlockingFeedStorage:
@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(self, uri, _stdout=None):
def __init__(self, uri, _stdout=None, *, feed_options=None):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
if feed_options and feed_options.get('overwrite', False) is True:
logger.warning('Standard output (stdout) storage does not support '
'overwriting. To suppress this warning, remove the '
'overwrite option from your FEEDS setting, or set '
'it to False.')
def open(self, spider):
return self._stdout
@ -78,14 +101,16 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri):
def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, 'ab')
return open(self.path, self.write_mode)
def store(self, file):
file.close()
@ -93,24 +118,8 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None):
# BEGIN Backward compatibility for initialising without keys (and
# without using from_crawler)
no_defaults = access_key is None and secret_key is None
if no_defaults:
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
warnings.warn(
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
"without AWS keys is deprecated. Please supply credentials or "
"use the `from_crawler()` constructor.",
category=ScrapyDeprecationWarning,
stacklevel=2
)
access_key = settings['AWS_ACCESS_KEY_ID']
secret_key = settings['AWS_SECRET_ACCESS_KEY']
# END Backward compatibility
def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
feed_options=None):
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
@ -127,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage):
else:
import boto
self.connect_s3 = boto.connect_s3
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
'option from your FEEDS setting or set it to True.')
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
uri=uri,
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
feed_options=feed_options,
)
def _store_in_thread(self, file):
@ -151,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage):
kwargs = {'policy': self.acl} if self.acl else {}
key.set_contents_from_file(file, **kwargs)
key.close()
file.close()
class GCSFeedStorage(BlockingFeedStorage):
@ -181,39 +197,45 @@ class GCSFeedStorage(BlockingFeedStorage):
class FTPFeedStorage(BlockingFeedStorage):
def __init__(self, uri, use_active_mode=False):
def __init__(self, uri, use_active_mode=False, *, feed_options=None):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.username = u.username
self.password = unquote(u.password)
self.password = unquote(u.password or '')
self.path = u.path
self.use_active_mode = use_active_mode
self.overwrite = not feed_options or feed_options.get('overwrite', True)
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
uri=uri,
use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
feed_options=feed_options,
)
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
port=self.port, username=self.username,
password=self.password, use_active_mode=self.use_active_mode
password=self.password, use_active_mode=self.use_active_mode,
overwrite=self.overwrite,
)
class _FeedSlot:
def __init__(self, file, exporter, storage, uri, format, store_empty):
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
self.file = file
self.exporter = exporter
self.storage = storage
# feed params
self.uri = uri
self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
self.uri_template = uri_template
self.uri = uri
# flags
self.itemcount = 0
self._exporting = False
@ -256,77 +278,126 @@ class FeedExporter:
category=ScrapyDeprecationWarning, stacklevel=2,
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed in self.settings.getdict('FEEDS').items():
for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
if not self._exporter_supported(feed['format']):
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed_options['format']):
raise NotConfigured
def open_spider(self, spider):
for uri, feed in self.feeds.items():
uri = uri % self._get_uri_params(spider, feed['uri_params'])
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
format=feed['format'],
fields_to_export=feed['fields'],
encoding=feed['encoding'],
indent=feed['indent'],
)
slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
self.slots.append(slot)
if slot.store_empty:
slot.start_exporting()
for uri, feed_options in self.feeds.items():
uri_params = self._get_uri_params(spider, feed_options['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
feed_options=feed_options,
spider=spider,
uri_template=uri,
))
def close_spider(self, spider):
deferred_list = []
for slot in self.slots:
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
d = defer.maybeDeferred(slot.storage.store, slot.file)
deferred_list.append(d)
continue
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
)
d = self._close_slot(slot, spider)
deferred_list.append(d)
return defer.DeferredList(deferred_list) if deferred_list else None
def _close_slot(self, slot, spider):
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
return defer.maybeDeferred(slot.storage.store, slot.file)
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
)
return d
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed_options: dict with parameters of feed
:param spider: user spider
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
"""
storage = self._get_storage(uri, feed_options)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
format=feed_options['format'],
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
)
slot = _FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
format=feed_options['format'],
store_empty=feed_options['store_empty'],
batch_id=batch_id,
uri_template=uri_template,
)
if slot.store_empty:
slot.start_exporting()
return slot
def item_scraped(self, item, spider):
slots = []
for slot in self.slots:
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
if (
self.feeds[slot.uri_template]['batch_item_count']
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
))
else:
slots.append(slot)
self.slots = slots
def _load_components(self, setting_prefix):
conf = without_none_values(self.settings.getwithbase(setting_prefix))
@ -343,11 +414,27 @@ class FeedExporter:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
def _storage_supported(self, uri):
def _settings_are_valid(self):
"""
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
"""
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
logger.error(
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
''.format(uri_template)
)
return False
return True
def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
try:
self._get_storage(uri)
self._get_storage(uri, feed_options)
return True
except NotConfigured as e:
logger.error("Disabled feed storage scheme: %(scheme)s. "
@ -365,15 +452,39 @@ class FeedExporter:
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
def _get_storage(self, uri):
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
def _get_storage(self, uri, feed_options):
"""Fork of create_instance specific to feed storage classes
def _get_uri_params(self, spider, uri_params):
It supports not passing the *feed_options* parameters to classes that
do not support it, and issuing a deprecation warning instead.
"""
feedcls = self.storages[urlparse(uri).scheme]
crawler = getattr(self, 'crawler', None)
def build_instance(builder, *preargs):
return build_storage(builder, uri, preargs=preargs)
if crawler and hasattr(feedcls, 'from_crawler'):
instance = build_instance(feedcls.from_crawler, crawler)
method_name = 'from_crawler'
elif hasattr(feedcls, 'from_settings'):
instance = build_instance(feedcls.from_settings, self.settings)
method_name = 'from_settings'
else:
instance = build_instance(feedcls)
method_name = '__new__'
if instance is None:
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
return instance
def _get_uri_params(self, spider, uri_params, slot=None):
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
params['time'] = ts
utc_now = datetime.utcnow()
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
params['batch_time'] = utc_now.isoformat().replace(':', '-')
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
uripar_function(params, spider)
return params

10
scrapy/extensions/httpcache.py
View File

@ -223,7 +223,7 @@ class DbmCacheStorage:
self.db = None
def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
@ -251,13 +251,13 @@ class DbmCacheStorage:
'headers': dict(response.headers),
'body': response.body,
}
self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
self.db['%s_time' % key] = str(time())
self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
key = self._request_key(request)
db = self.db
tkey = '%s_time' % key
tkey = f'{key}_time'
if tkey not in db:
return # not found
@ -265,7 +265,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return # expired
return pickle.loads(db['%s_data' % key])
return pickle.loads(db[f'{key}_data'])
def _request_key(self, request):
return request_fingerprint(request)

2
scrapy/extensions/memdebug.py
View File

@ -30,4 +30,4 @@ class MemoryDebugger:
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)

14
scrapy/extensions/memusage.py
View File

@ -82,8 +82,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
"%s terminated: memory usage exceeded %dM at %s"
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}M at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1)
@ -105,8 +105,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
"%s warning: memory usage reached %dM at %s"
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}M at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1)
@ -115,9 +115,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"

10
scrapy/extensions/statsmailer.py
View File

@ -24,11 +24,11 @@ class StatsMailer:
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider):
spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n"
body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items())
body += "\n\n%s stats\n\n" % spider.name
body += "\n".join("%-50s : %s" % i for i in spider_stats.items())
return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body)
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
body += f"\n\n{spider.name} stats\n\n"
body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items())
return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body)

2
scrapy/http/common.py
View File

@ -1,6 +1,6 @@
def obsolete_setter(setter, attrname):
def newsetter(self, value):
c = self.__class__.__name__
msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c)
msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
raise AttributeError(msg)
return newsetter

10
scrapy/http/headers.py
View File

@ -8,7 +8,7 @@ class Headers(CaselessDict):
def __init__(self, seq=None, encoding='utf-8'):
self.encoding = encoding
super(Headers, self).__init__(seq)
super().__init__(seq)
def normkey(self, key):
"""Normalize key to bytes"""
@ -33,23 +33,23 @@ class Headers(CaselessDict):
elif isinstance(x, int):
return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key):
try:
return super(Headers, self).__getitem__(key)[-1]
return super().__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
return super(Headers, self).get(key, def_val)[-1]
return super().get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
return super(Headers, self).__getitem__(key)
return super().__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)

12
scrapy/http/request/__init__.py
View File

@ -25,13 +25,13 @@ class Request(object_ref):
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
raise TypeError("Request priority not an integer: %r" % priority)
raise TypeError(f"Request priority not an integer: {priority!r}")
self.priority = priority
if callback is not None and not callable(callback):
raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
raise TypeError(f'callback must be a callable, got {type(callback).__name__}')
if errback is not None and not callable(errback):
raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
raise TypeError(f'errback must be a callable, got {type(errback).__name__}')
self.callback = callback
self.errback = errback
@ -60,13 +60,13 @@ class Request(object_ref):
def _set_url(self, url):
if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}')
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
if ('://' not in self._url) and (not self._url.startswith('data:')):
raise ValueError('Missing scheme in request url: %s' % self._url)
raise ValueError(f'Missing scheme in request url: {self._url}')
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -86,7 +86,7 @@ class Request(object_ref):
return self._encoding
def __str__(self):
return "<%s %s>" % (self.method, self.url)
return f"<{self.method} {self.url}>"
__repr__ = __str__

25
scrapy/http/request/form.py
View File

@ -24,7 +24,7 @@ class FormRequest(Request):
if formdata and kwargs.get('method') is None:
kwargs['method'] = 'POST'
super(FormRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata
@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath):
base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)
raise ValueError(f"No <form> element found in {response}")
if formname is not None:
f = root.xpath('//form[@name="%s"]' % formname)
f = root.xpath(f'//form[@name="{formname}"]')
if f:
return f[0]
if formid is not None:
f = root.xpath('//form[@id="%s"]' % formid)
f = root.xpath(f'//form[@id="{formid}"]')
if f:
return f[0]
@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
el = el.getparent()
if el is None:
break
raise ValueError('No <form> element found with %s' % formxpath)
raise ValueError(f'No <form> element found with {formxpath}')
# If we get here, it means that either formname was None
# or invalid
@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
try:
form = forms[formnumber]
except IndexError:
raise IndexError("Form number %d not found in %s" %
(formnumber, response))
raise IndexError(f"Form number {formnumber} not found in {response}")
else:
return form
@ -133,7 +132,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
namespaces={
"re": "http://exslt.org/regular-expressions"})
values = [(k, u'' if v is None else v)
values = [(k, '' if v is None else v)
for k, v in (_value(e) for e in inputs)
if k and k not in formdata_keys]
@ -168,7 +167,7 @@ def _select_value(ele, n, v):
# This is a workround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath('.//option[@selected]')
v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
return n, v
@ -205,12 +204,12 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
elif len(el) > 1:
raise ValueError("Multiple elements found (%r) matching the criteria "
"in clickdata: %r" % (el, clickdata))
raise ValueError(f"Multiple elements found ({el!r}) matching the "
f"criteria in clickdata: {clickdata!r}")
else:
raise ValueError('No clickable element matching clickdata: %r' % (clickdata,))
raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')

4
scrapy/http/request/json_request.py
View File

@ -32,7 +32,7 @@ class JsonRequest(Request):
if 'method' not in kwargs:
kwargs['method'] = 'POST'
super(JsonRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@ -47,7 +47,7 @@ class JsonRequest(Request):
elif not body_passed and data_passed:
kwargs['body'] = self._dumps(data)
return super(JsonRequest, self).replace(*args, **kwargs)
return super().replace(*args, **kwargs)
def _dumps(self, data):
"""Convert to JSON """

2
scrapy/http/request/rpc.py
View File

@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
if encoding is not None:
kwargs['encoding'] = encoding
super(XmlRpcRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'text/xml')

6
scrapy/http/response/__init__.py
View File

@ -55,8 +55,8 @@ class Response(object_ref):
if isinstance(url, str):
self._url = url
else:
raise TypeError('%s url must be str, got %s:' %
(type(self).__name__, type(url).__name__))
raise TypeError(f'{type(self).__name__} url must be str, '
f'got {type(url).__name__}')
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -77,7 +77,7 @@ class Response(object_ref):
body = property(_get_body, obsolete_setter(_set_body, 'body'))
def __str__(self):
return "<%d %s>" % (self.status, self.url)
return f"<{self.status} {self.url}>"
__repr__ = __str__

25
scrapy/http/response/text.py
View File

@ -35,23 +35,23 @@ class TextResponse(Response):
self._cached_benc = None
self._cached_ubody = None
self._cached_selector = None
super(TextResponse, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
super()._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
raise TypeError('Cannot convert unicode body - '
f'{type(self).__name__} has no encoding')
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
super()._set_body(body)
def replace(self, *args, **kwargs):
kwargs.setdefault('encoding', self.encoding)
@ -92,7 +92,7 @@ class TextResponse(Response):
# _body_inferred_encoding is called
benc = self.encoding
if self._cached_ubody is None:
charset = 'charset=%s' % benc
charset = f'charset={benc}'
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
@ -166,7 +166,7 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(
return super().follow(
url=url,
callback=callback,
method=method,
@ -226,7 +226,7 @@ class TextResponse(Response):
for sel in selectors:
with suppress(_InvalidSelector):
urls.append(_url_from_selector(sel))
return super(TextResponse, self).follow_all(
return super().follow_all(
urls=urls,
callback=callback,
method=method,
@ -255,12 +255,11 @@ def _url_from_selector(sel):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise _InvalidSelector("Unsupported selector: %s" % sel)
raise _InvalidSelector(f"Unsupported selector: {sel}")
if sel.root.tag not in ('a', 'link'):
raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
sel.root.tag)
raise _InvalidSelector("Only <a> and <link> elements are supported; "
f"got <{sel.root.tag}>")
href = sel.root.get('href')
if href is None:
raise _InvalidSelector("<%s> element has no href attribute: %s" %
(sel.root.tag, sel))
raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}")
return strip_html5_whitespace(href)

16
scrapy/item.py
View File

@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
class Field(dict):
@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
def __new__(mcs, class_name, bases, attrs):
classcell = attrs.pop('__classcell__', None)
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
_class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
fields = getattr(_class, 'fields', {})
new_attrs = {}
@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
new_attrs['_class'] = _class
if classcell is not None:
new_attrs['__classcell__'] = classcell
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
return super().__new__(mcs, class_name, bases, new_attrs)
class DictItem(MutableMapping, BaseItem):
@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(DictItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
self._values = {}
@ -96,20 +96,20 @@ class DictItem(MutableMapping, BaseItem):
if key in self.fields:
self._values[key] = value
else:
raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
raise KeyError(f"{self.__class__.__name__} does not support field: {key}")
def __delitem__(self, key):
del self._values[key]
def __getattr__(self, name):
if name in self.fields:
raise AttributeError("Use item[%r] to get field value" % name)
raise AttributeError(f"Use item[{name!r}] to get field value")
raise AttributeError(name)
def __setattr__(self, name, value):
if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
super(DictItem, self).__setattr__(name, value)
raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
super().__setattr__(name, value)
def __len__(self):
return len(self._values)

6
scrapy/link.py
View File

@ -14,7 +14,7 @@ class Link:
def __init__(self, url, text='', fragment='', nofollow=False):
if not isinstance(url, str):
got = url.__class__.__name__
raise TypeError("Link urls must be str objects, got %s" % got)
raise TypeError(f"Link urls must be str objects, got {got}")
self.url = url
self.text = text
self.fragment = fragment
@ -33,6 +33,6 @@ class Link:
def __repr__(self):
return (
'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
% (self.url, self.text, self.fragment, self.nofollow)
f'Link(url={self.url!r}, text={self.text!r}, '
f'fragment={self.fragment!r}, nofollow={self.nofollow!r})'
)

2
scrapy/linkextractors/__init__.py
View File

@ -65,7 +65,7 @@ class FilteringLinkExtractor:
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(FilteringLinkExtractor, cls).__new__(cls)
return super().__new__(cls)
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):

4
scrapy/linkextractors/lxmlhtml.py
View File

@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
url = safe_url_string(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
link = Link(url, _collect_string_content(el) or '',
nofollow=rel_has_nofollow(el.get('rel')))
links.append(link)
return self._deduplicate_if_needed(links)
@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
strip=strip,
canonicalized=canonicalize
)
super(LxmlLinkExtractor, self).__init__(
super().__init__(
link_extractor=lx,
allow=allow,
deny=deny,

Some files were not shown because too many files have changed in this diff Show More