RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into allow-customizing-export-column-names

This commit is contained in:
Adrián Chaves 2020-11-20 20:06:18 +01:00 committed by GitHub
parent 5138f9a965 e17c890be1
commit 07379cf9b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
217 changed files with 4153 additions and 2017 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.3.0
current_version = 2.4.1
commit = True
tag = True
tag_name = {new_version}

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
tests/sample_data/** binary

31
.github/workflows/main.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: Run test suite
on: [push, pull_request]
jobs:
test-windows:
name: "Windows Tests"
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest]
python-version: [3.7, 3.8]
env: [TOXENV: py]
include:
- os: windows-latest
python-version: 3.6
env:
TOXENV: windows-pinned
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Run test suite
env: ${{ matrix.env }}
run: |
pip install -U tox twine wheel codecov
tox

2
.gitignore vendored
View File

@ -16,6 +16,8 @@ htmlcov/
.coverage.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
/tests/keys/localhost.key
# Windows
Thumbs.db

14
.travis.yml
View File

@ -19,16 +19,10 @@ matrix:
python: 3.8
- env: TOXENV=pinned
python: 3.5.2
python: 3.6.1
- env: TOXENV=asyncio-pinned
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
- env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
- env: TOXENV=py
python: 3.5
- env: TOXENV=asyncio
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
python: 3.6.1
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
- env: TOXENV=py
python: 3.6
@ -50,7 +44,7 @@ install:
- |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"

2
README.rst
View File

@ -40,7 +40,7 @@ including a list of features.
Requirements
============
* Python 3.5.2+
* Python 3.6+
* Works on Linux, Windows, macOS, BSD
Install

25
appveyor.yml
View File

@ -1,25 +0,0 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

24
azure-pipelines.yml
View File

@ -1,24 +0,0 @@
variables:
TOXENV: py
pool:
vmImage: 'windows-latest'
strategy:
matrix:
Python35:
python.version: '3.5'
TOXENV: windows-pinned
Python36:
python.version: '3.6'
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- script: |
pip install -U tox twine wheel codecov
tox
displayName: 'Run test suite'

8
conftest.py
View File

@ -2,6 +2,8 @@ from pathlib import Path
import pytest
from tests.keys import generate_keys
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
@ -14,8 +16,6 @@ collect_ignore = [
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):
@ -53,3 +53,7 @@ def reactor_pytest(request):
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
# Generate localhost certificate files, needed by some tests
generate_keys()

2
docs/conf.py
View File

@ -49,7 +49,7 @@ master_doc = 'index'
# General information about the project.
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the

18
docs/contributing.rst
View File

@ -108,6 +108,11 @@ Well-written patches should:
tox -e docs-coverage
* if you are removing deprecated code, first make sure that at least 1 year
(12 months) has passed since the release that introduced the deprecation.
See :ref:`deprecation-policy`.
.. _submitting-patches:
Submitting patches
@ -135,7 +140,7 @@ original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
not considered rude as long as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
@ -194,6 +199,17 @@ In any case, if something is covered in a docstring, use the
documentation instead of duplicating the docstring in files within the
``docs/`` directory.
Documentation updates that cover new or modified features must use Sphinx’s
:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use
``VERSION`` as version, we will replace it with the actual version right before
the corresponding release. When we release a new major or minor version of
Scrapy, we remove these directives if they are older than 3 years.
Documentation about deprecated features must be removed as those features are
deprecated, so that new readers do not run into it. New deprecations and
deprecation removals are documented in the :ref:`release notes <news>`.
Tests
=====

6
docs/faq.rst
View File

@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
To dump into a JSON file::
scrapy crawl myspider -o items.json
scrapy crawl myspider -O items.json
To dump into a CSV file::
scrapy crawl myspider -o items.csv
scrapy crawl myspider -O items.csv
To dump into a XML file::
scrapy crawl myspider -o items.xml
scrapy crawl myspider -O items.xml
For more information see :ref:`topics-feed-exports`

1
docs/index.rst
View File

@ -78,7 +78,6 @@ Basic concepts
topics/settings
topics/exceptions
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.

8
docs/intro/install.rst
View File

@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
the PyPy 5.9+ implementation (see :ref:`python:implementations`).
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
Installing Scrapy
@ -211,8 +211,8 @@ PyPy
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,

26
docs/intro/overview.rst
View File

@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
if next_page is not None:
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.json
scrapy runspider quotes_spider.py -o quotes.jl
When this finishes you will have in the ``quotes.jl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
When this finishes you will have in the ``quotes.json`` file a list of the
quotes in JSON format, containing text and author, looking like this (reformatted
here for better readability)::
[{
"author": "Jane Austen",
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
},
{
"author": "Groucho Marx",
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
},
{
"author": "Steve Martin",
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
},
...]
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
{"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
...
What just happened?

21
docs/intro/tutorial.rst
View File

@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
self.log('Saved file %s' % filename)
self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
@ -190,7 +190,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
@ -405,8 +405,6 @@ to get all of them:
from sys import version_info
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
@ -464,16 +462,15 @@ Storing the scraped data
The simplest way to store the scraped data is by using :ref:`Feed exports
<topics-feed-exports>`, with the following command::
scrapy crawl quotes -o quotes.json
scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also use other formats, like `JSON Lines`_::
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@ -704,7 +701,7 @@ Using spider arguments
You can provide command line arguments to your spiders by using the ``-a``
option when running them::
scrapy crawl quotes -o quotes-humor.json -a tag=humor
scrapy crawl quotes -O quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.

329
docs/news.rst
View File

@ -3,6 +3,333 @@
Release notes
=============
.. _release-2.4.1:
Scrapy 2.4.1 (2020-11-17)
-------------------------
- Fixed :ref:`feed exports <topics-feed-exports>` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`)
- Fixed the AsyncIO event loop handling, which could make code hang
(:issue:`4855`, :issue:`4872`)
- Fixed the IPv6-capable DNS resolver
:class:`~scrapy.resolver.CachingHostnameResolver` for download handlers
that call
:meth:`reactor.resolve <twisted.internet.interfaces.IReactorCore.resolve>`
(:issue:`4802`, :issue:`4803`)
- Fixed the output of the :command:`genspider` command showing placeholders
instead of the import path of the generated spider module (:issue:`4874`)
- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
:issue:`4876`)
.. _release-2.4.0:
Scrapy 2.4.0 (2020-10-11)
-------------------------
Highlights:
* Python 3.5 support has been dropped.
* The ``file_path`` method of :ref:`media pipelines <topics-media-pipeline>`
can now access the source :ref:`item <topics-items>`.
This allows you to set a download file path based on item data.
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
to define keyword parameters to pass to :ref:`item exporter classes
<topics-exporters>`
* You can now choose whether :ref:`feed exports <topics-feed-exports>`
overwrite or append to the output file.
For example, when using the :command:`crawl` or :command:`runspider`
commands, you can use the ``-O`` option instead of ``-o`` to overwrite the
output file.
* Zstd-compressed responses are now supported if zstandard_ is installed.
* In settings, where the import path of a class is required, it is now
possible to pass a class object instead.
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
* Python 3.6 or greater is now required; support for Python 3.5 has been
dropped
As a result:
- When using PyPy, PyPy 7.2.0 or greater :ref:`is now required
<faq-python-versions>`
- For Amazon S3 storage support in :ref:`feed exports
<topics-feed-storage-s3>` or :ref:`media pipelines
<media-pipelines-s3>`, botocore_ 1.4.87 or greater is now required
- To use the :ref:`images pipeline <images-pipeline>`, Pillow_ 4.0.0 or
greater is now required
(:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`,
:issue:`4764`)
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again
discards cookies defined in :attr:`Request.headers
<scrapy.http.Request.headers>`.
We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it
was reported that the current implementation could break existing code.
If you need to set cookies for a request, use the :class:`Request.cookies
<scrapy.http.Request>` parameter.
A future version of Scrapy will include a new, better implementation of the
reverted bug fix.
(:issue:`4717`, :issue:`4823`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the
values of ``access_key`` and ``secret_key`` from the running project
settings when they are not passed to its ``__init__`` method; you must
either pass those parameters to its ``__init__`` method or use
:class:`S3FeedStorage.from_crawler
<scrapy.extensions.feedexport.S3FeedStorage.from_crawler>`
(:issue:`4356`, :issue:`4411`, :issue:`4688`)
* :attr:`Rule.process_request <scrapy.spiders.crawl.Rule.process_request>`
no longer admits callables which expect a single ``request`` parameter,
rather than both ``request`` and ``response`` (:issue:`4818`)
Deprecations
~~~~~~~~~~~~
* In custom :ref:`media pipelines <topics-media-pipeline>`, signatures that
do not accept a keyword-only ``item`` parameter in any of the methods that
:ref:`now support this parameter <media-pipeline-item-parameter>` are now
deprecated (:issue:`4628`, :issue:`4686`)
* In custom :ref:`feed storage backend classes <topics-feed-storage>`,
``__init__`` method signatures that do not accept a keyword-only
``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`,
:issue:`4512`)
* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated
(:issue:`4684`, :issue:`4701`)
* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use
:func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`,
:issue:`4776`)
New features
~~~~~~~~~~~~
.. _media-pipeline-item-parameter:
* The following methods of :ref:`media pipelines <topics-media-pipeline>` now
accept an ``item`` keyword-only parameter containing the source
:ref:`item <topics-items>`:
- In :class:`scrapy.pipelines.files.FilesPipeline`:
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded`
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_path`
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download`
- In :class:`scrapy.pipelines.images.ImagesPipeline`:
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download`
(:issue:`4628`, :issue:`4686`)
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
to define keyword parameters to pass to :ref:`item exporter classes
<topics-exporters>` (:issue:`4606`, :issue:`4768`)
* :ref:`Feed exports <topics-feed-exports>` gained overwrite support:
* When using the :command:`crawl` or :command:`runspider` commands, you
can use the ``-O`` option instead of ``-o`` to overwrite the output
file
* You can use the ``overwrite`` key in the :setting:`FEEDS` setting to
configure whether to overwrite the output file (``True``) or append to
its content (``False``)
* The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage
backend classes <topics-feed-storage>` now receive a new keyword-only
parameter, ``feed_options``, which is a dictionary of :ref:`feed
options <feed-options>`
(:issue:`547`, :issue:`716`, :issue:`4512`)
* Zstd-compressed responses are now supported if zstandard_ is installed
(:issue:`4831`)
* In settings, where the import path of a class is required, it is now
possible to pass a class object instead (:issue:`3870`, :issue:`3873`).
This includes also settings where only part of its value is made of an
import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or
:setting:`DOWNLOAD_HANDLERS`.
* :ref:`Downloader middlewares <topics-downloader-middleware>` can now
override :class:`response.request <scrapy.http.Response.request>`.
If a :ref:`downloader middleware <topics-downloader-middleware>` returns
a :class:`~scrapy.http.Response` object from
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
or
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
with a custom :class:`~scrapy.http.Request` object assigned to
:class:`response.request <scrapy.http.Response.request>`:
- The response is handled by the callback of that custom
:class:`~scrapy.http.Request` object, instead of being handled by the
callback of the original :class:`~scrapy.http.Request` object
- That custom :class:`~scrapy.http.Request` object is now sent as the
``request`` argument to the :signal:`response_received` signal, instead
of the original :class:`~scrapy.http.Request` object
(:issue:`4529`, :issue:`4632`)
* When using the :ref:`FTP feed storage backend <topics-feed-storage-ftp>`:
- It is now possible to set the new ``overwrite`` :ref:`feed option
<feed-options>` to ``False`` to append to an existing file instead of
overwriting it
- The FTP password can now be omitted if it is not necessary
(:issue:`547`, :issue:`716`, :issue:`4512`)
* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now
supports an ``errors`` parameter to indicate how to handle encoding errors
(:issue:`4755`)
* When :ref:`using asyncio <using-asyncio>`, it is now possible to
:ref:`set a custom asyncio loop <using-custom-loops>` (:issue:`4306`,
:issue:`4414`)
* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are
spider methods that delegate on other callable (:issue:`4756`)
* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged
message is now a warning, instead of an error (:issue:`3874`,
:issue:`3886`, :issue:`4752`)
Bug fixes
~~~~~~~~~
* The :command:`genspider` command no longer overwrites existing files
unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`,
:issue:`4623`)
* Cookies with an empty value are no longer considered invalid cookies
(:issue:`4772`)
* The :command:`runspider` command now supports files with the ``.pyw`` file
extension (:issue:`4643`, :issue:`4646`)
* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
middleware now simply ignores unsupported proxy values (:issue:`3331`,
:issue:`4778`)
* Checks for generator callbacks with a ``return`` statement no longer warn
about ``return`` statements in nested functions (:issue:`4720`,
:issue:`4721`)
* The system file mode creation mask no longer affects the permissions of
files generated using the :command:`startproject` command (:issue:`4722`)
* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
(:issue:`861`, :issue:`4746`)
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
work when using a headless browser (:issue:`4835`)
Documentation
~~~~~~~~~~~~~
* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`,
:issue:`4724`)
* Improved the documentation of
:ref:`link extractors <topics-link-extractors>` with an usage example from
a spider callback and reference documentation for the
:class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`)
* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the
:class:`~scrapy.extensions.closespider.CloseSpider` extension
(:issue:`4836`)
* Removed references to Python 2’s ``unicode`` type (:issue:`4547`,
:issue:`4703`)
* We now have an :ref:`official deprecation policy <deprecation-policy>`
(:issue:`4705`)
* Our :ref:`documentation policies <documentation-policies>` now cover usage
of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged`
directives, and we have removed usages referencing Scrapy 1.4.0 and earlier
versions (:issue:`3971`, :issue:`4310`)
* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`,
:issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Extended typing hints (:issue:`4243`, :issue:`4691`)
* Added tests for the :command:`check` command (:issue:`4663`)
* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`)
* Improved Windows test coverage (:issue:`4723`)
* Switched to :ref:`formatted string literals <f-strings>` where possible
(:issue:`4307`, :issue:`4324`, :issue:`4672`)
* Modernized :func:`super` usage (:issue:`4707`)
* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`,
:issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`,
:issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`,
:issue:`4820`, :issue:`4822`, :issue:`4839`)
.. _release-2.3.0:
Scrapy 2.3.0 (2020-08-04)
@ -4008,9 +4335,9 @@ First release of Scrapy.
.. _six: https://six.readthedocs.io/
.. _tox: https://pypi.org/project/tox/
.. _Twisted: https://twistedmatrix.com/trac/
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _w3lib: https://github.com/scrapy/w3lib
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
.. _Zsh: https://www.zsh.org/
.. _zstandard: https://pypi.org/project/zstandard/

2
docs/topics/api.rst
View File

@ -4,8 +4,6 @@
Core API
========
.. versionadded:: 0.15
This section documents the Scrapy core API, and it's intended for developers of
extensions and middlewares.

14
docs/topics/asyncio.rst
View File

@ -1,3 +1,5 @@
.. _using-asyncio:
=======
asyncio
=======
@ -26,3 +28,15 @@ reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.

2
docs/topics/autothrottle.rst
View File

@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies.
AUTOTHROTTLE_TARGET_CONCURRENCY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.1
Default: ``1.0``
Average number of requests Scrapy should be sending in parallel to remote

2
docs/topics/benchmarking.rst
View File

@ -4,8 +4,6 @@
Benchmarking
============
.. versionadded:: 0.17
Scrapy comes with a simple benchmarking suite that spawns a local HTTP server
and crawls it at the maximum possible speed. The goal of this benchmarking is
to get an idea of how Scrapy performs in your hardware, in order to have a

4
docs/topics/commands.rst
View File

@ -6,8 +6,6 @@
Command line tool
=================
.. versionadded:: 0.10
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
just call "commands" or "Scrapy commands".
@ -566,8 +564,6 @@ and Platform info, which is useful for bug reports.
bench
-----
.. versionadded:: 0.17
* Syntax: ``scrapy bench``
* Requires project: *no*

4
docs/topics/contracts.rst
View File

@ -4,8 +4,6 @@
Spiders Contracts
=================
.. versionadded:: 0.15
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@ -81,7 +79,7 @@ override three methods:
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function
:type method: collections.abc.Callable
:param args: list of arguments passed into the docstring (whitespace
separated)

15
docs/topics/coroutines.rst
View File

@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
The following are known caveats of the current implementation that we aim
to address in future versions of Scrapy:
- The callback output is not processed until the whole callback finishes.
.. note:: The callback output is not processed until the whole callback
finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
- Because `asynchronous generators were introduced in Python 3.6`_, you
can only use ``yield`` if you are using Python 3.6 or later.
If you need to output multiple items or requests and you are using
Python 3.5, return an iterable (e.g. a list) instead.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
Usage
=====

138
docs/topics/developer-tools.rst
View File

@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping
=================================================
Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with
to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser.
guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_.
@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things:
Inspecting a website
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click
Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this:
@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote:
.. code-block:: html
@ -121,7 +121,7 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
@ -130,45 +130,45 @@ the `has-class-extension`_:
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line.
we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the
The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate
a use case:
a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites.
can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool:
The Network-tool
================
While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_.
take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell:
.. skip: next
@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell:
(...)
>>> view(response)
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
.. image:: _images/network_01.png
:width: 777
@ -189,21 +189,21 @@ bar with the word ``Loading...``.
:alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``.
than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six
new requests.
new requests.
.. image:: _images/network_02.png
:width: 777
@ -212,31 +212,31 @@ new requests.
Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out
where our quotes are coming from:
where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png
:width: 777
:height: 375
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
import scrapy
@ -255,17 +255,17 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading
If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
into our ``url``.
.. _requests-from-curl:
@ -298,7 +298,7 @@ Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.

48
docs/topics/downloader-middleware.rst
View File

@ -207,6 +207,11 @@ CookiesMiddleware
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`
@ -217,8 +222,6 @@ The following settings can be used to configure the cookie middleware:
Multiple cookie sessions per spider
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 0.15
There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
@ -475,8 +478,6 @@ DBM storage backend
.. class:: DbmCacheStorage
.. versionadded:: 0.13
A DBM_ storage backend is also available for the HTTP cache middleware.
By default, it uses the :mod:`dbm`, but you can change it with the
@ -549,15 +550,10 @@ settings:
HTTPCACHE_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.11
Default: ``False``
Whether the HTTP cache will be enabled.
.. versionchanged:: 0.11
Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
.. setting:: HTTPCACHE_EXPIRATION_SECS
HTTPCACHE_EXPIRATION_SECS
@ -570,9 +566,6 @@ Expiration time for cached requests, in seconds.
Cached requests older than this time will be re-downloaded. If zero, cached
requests will never expire.
.. versionchanged:: 0.11
Before 0.11, zero meant cached requests always expire.
.. setting:: HTTPCACHE_DIR
HTTPCACHE_DIR
@ -589,8 +582,6 @@ project data dir. For more info see: :ref:`topics-project-structure`.
HTTPCACHE_IGNORE_HTTP_CODES
^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``[]``
Don't cache response with these HTTP codes.
@ -609,8 +600,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade
HTTPCACHE_IGNORE_SCHEMES
^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``['file']``
Don't cache responses with these URI schemes.
@ -629,8 +618,6 @@ The class which implements the cache storage backend.
HTTPCACHE_DBM_MODULE
^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
@ -641,8 +628,6 @@ The database module to use in the :ref:`DBM storage backend
HTTPCACHE_POLICY
^^^^^^^^^^^^^^^^
.. versionadded:: 0.18
Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
The class which implements the cache policy.
@ -652,8 +637,6 @@ The class which implements the cache policy.
HTTPCACHE_GZIP
^^^^^^^^^^^^^^
.. versionadded:: 1.0
Default: ``False``
If enabled, will compress all cached data with gzip.
@ -664,8 +647,6 @@ This setting is specific to the Filesystem backend.
HTTPCACHE_ALWAYS_STORE
^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``False``
If enabled, will cache pages unconditionally.
@ -684,8 +665,6 @@ responses you feed to the cache middleware.
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``[]``
List of Cache-Control directives in responses to be ignored.
@ -710,11 +689,14 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses,
provided `brotlipy`_ is installed.
This middleware also supports decoding `brotli-compressed`_ as well as
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -735,8 +717,6 @@ HttpProxyMiddleware
.. module:: scrapy.downloadermiddlewares.httpproxy
:synopsis: Http Proxy Middleware
.. versionadded:: 0.8
.. reqmeta:: proxy
.. class:: HttpProxyMiddleware
@ -817,8 +797,6 @@ RedirectMiddleware settings
REDIRECT_ENABLED
^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Redirect middleware will be enabled.
@ -860,8 +838,6 @@ MetaRefreshMiddleware settings
METAREFRESH_ENABLED
^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.17
Default: ``True``
Whether the Meta Refresh middleware will be enabled.
@ -924,8 +900,6 @@ RetryMiddleware Settings
RETRY_ENABLED
^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Retry middleware will be enabled.
@ -1179,8 +1153,6 @@ AjaxCrawlMiddleware Settings
AJAXCRAWL_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.21
Default: ``False``
Whether the AjaxCrawlMiddleware will be enabled. You may want to

14
docs/topics/email.rst
View File

@ -62,10 +62,10 @@ rest of the framework.
:type smtpport: int
:param smtptls: enforce using SMTP STARTTLS
:type smtptls: boolean
:type smtptls: bool
:param smtpssl: enforce using a secure SSL connection
:type smtpssl: boolean
:type smtpssl: bool
.. classmethod:: from_settings(settings)
@ -79,14 +79,14 @@ rest of the framework.
Send email to the given recipients.
:param to: the e-mail recipients
:type to: str or list of str
:param to: the e-mail recipients as a string or as a list of strings
:type to: str or list
:param subject: the subject of the e-mail
:type subject: str
:param cc: the e-mails to CC
:type cc: str or list of str
:param cc: the e-mails to CC as a string or as a list of strings
:type cc: str or list
:param body: the e-mail body
:type body: str
@ -96,7 +96,7 @@ rest of the framework.
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
attachment and ``file_object`` is a readable file object with the
contents of the attachment
:type attachs: iterable
:type attachs: collections.abc.Iterable
:param mimetype: the MIME type of the e-mail
:type mimetype: str

15
docs/topics/exporters.rst
View File

@ -57,7 +57,7 @@ value of one of their fields::
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
f = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(f)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
@ -98,7 +98,7 @@ Example::
import scrapy
def serialize_price(value):
return '$ %s' % str(value)
return f'$ {str(value)}'
class Product(scrapy.Item):
name = scrapy.Field()
@ -122,7 +122,7 @@ Example::
def serialize_field(self, field, name, value):
if field == 'price':
return '$ %s' % str(value)
return f'$ {str(value)}'
return super(Product, self).serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -308,7 +308,7 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
@ -321,12 +321,17 @@ CsvItemExporter
:param include_headers_line: If enabled, makes the exporter output a header
line with the field names taken from
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
:type include_headers_line: boolean
:type include_headers_line: bool
:param join_multivalued: The char (or chars) that will be used for joining
multi-valued fields, if found.
:type include_headers_line: str
:param errors: The optional string that specifies how encoding and decoding
errors are to be handled. For more information see
:class:`io.TextIOWrapper`.
:type errors: str
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
:func:`csv.writer` function, so you can use any :func:`csv.writer` function

12
docs/topics/extensions.rst
View File

@ -257,6 +257,12 @@ settings:
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
When a certain closing condition is met, requests which are
currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS`
requests) are still processed.
.. setting:: CLOSESPIDER_TIMEOUT
CLOSESPIDER_TIMEOUT
@ -279,8 +285,6 @@ Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -288,8 +292,6 @@ If zero (or non set), spiders won't be closed by number of passed items.
CLOSESPIDER_PAGECOUNT
"""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of responses to crawl. If the spider
@ -302,8 +304,6 @@ number of crawled responses.
CLOSESPIDER_ERRORCOUNT
""""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of errors to receive before

143
docs/topics/feed-exports.rst
View File

@ -4,8 +4,6 @@
Feed exports
============
.. versionadded:: 0.10
One of the most frequently required features when implementing scrapers is
being able to store the scraped data properly and, quite often, that means
generating an "export file" with the scraped data (commonly called "export
@ -186,7 +184,7 @@ The feeds are stored on `Amazon S3`_.
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
* Required external libraries: `botocore`_
* Required external libraries: `botocore`_ >= 1.4.87
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -291,6 +289,7 @@ Default: ``{}``
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
This setting is required for enabling the feed export feature.
See :ref:`topics-feed-storage-backends` for supported URI schemes.
@ -304,6 +303,9 @@ For instance::
'store_empty': False,
'fields': None,
'indent': 4,
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
@ -317,17 +319,54 @@ For instance::
},
}
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition.
.. _feed-options:
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition:
- ``format``: the :ref:`serialization format <topics-feed-format>`.
This setting is mandatory, there is no fallback value.
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
.. versionadded:: 2.4.0
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
The default value depends on the :ref:`storage backend
<topics-feed-storage-backends>`:
- :ref:`topics-feed-storage-fs`: ``False``
- :ref:`topics-feed-storage-ftp`: ``True``
.. note:: Some FTP servers may not support appending to files (the
``APPE`` FTP command).
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
* ``format``: the serialization format to be used for the feed.
See :ref:`topics-feed-format` for possible values.
Mandatory, no fallback setting
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. setting:: FEED_EXPORT_ENCODING
@ -477,7 +516,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
-----------------------------
----------------------------
.. versionadded:: 2.3.0
Default: ``0``
@ -491,7 +532,7 @@ generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
alter the number format. For example, to make the batch ID a 5-digit
@ -508,16 +549,78 @@ And your :command:`crawl` command line is::
The command line above can generate a directory tree like::
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fewer.
.. setting:: FEED_URI_PARAMS
FEED_URI_PARAMS
---------------
Default: ``None``
A string with the import path of a function to set the parameters to apply with
:ref:`printf-style string formatting <python:old-string-formatting>` to the
feed URI.
The function signature should be as follows:
.. function:: uri_params(params, spider)
Return a :class:`dict` of key-value pairs to apply to the feed URI using
:ref:`printf-style string formatting <python:old-string-formatting>`.
:param params: default key-value pairs
Specifically:
- ``batch_id``: ID of the file batch. See
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
.. versionadded:: 2.3.0
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.spiders.Spider
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore

6
docs/topics/item-pipeline.rst
View File

@ -96,7 +96,7 @@ contain a price::
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
@ -211,7 +211,7 @@ item.
# Save screenshot to file, filename will be hash of url.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
@ -240,7 +240,7 @@ returns multiples items with the same id::
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item)
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
return item

2
docs/topics/jobs.rst
View File

@ -65,7 +65,7 @@ Cookies expiration
------------------
Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
scheduled may no longer work. This won't be an issue if your spider doesn't rely
on cookies.

4
docs/topics/leaks.rst
View File

@ -102,7 +102,7 @@ A real example
Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
:param ignore: if given, all objects from the specified class (or tuple of
classes) will be ignored.
:type ignore: class or classes tuple
:type ignore: type or tuple
.. function:: get_oldest(class_name)

35
docs/topics/link-extractors.rst
View File

@ -10,12 +10,19 @@ The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
list of matching :class:`~scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
def parse(self, response):
for link in self.link_extractor.extract_links(response):
yield Request(link.url, callback=self.parse)
.. _topics-link-extractors-ref:
@ -46,13 +53,13 @@ LxmlLinkExtractor
:param allow: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be extracted. If not
given (or empty), it will match all links.
:type allow: a regular expression (or list of)
:type allow: str or list
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
:type deny: str or list
:param allow_domains: a single value or a list of string containing
domains which will be considered for extracting the links
@ -88,7 +95,7 @@ LxmlLinkExtractor
that the link's text must match in order to be extracted. If not
given (or empty), it will match all links. If a list of regular expressions is
given, the link will be extracted if it matches at least one.
:type restrict_text: a regular expression (or list of)
:type restrict_text: str or list
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
@ -106,11 +113,11 @@ LxmlLinkExtractor
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:type canonicalize: bool
:param unique: whether duplicate filtering should be applied to extracted
links.
:type unique: boolean
:type unique: bool
:param process_value: a function which receives each value extracted from
the tag and attributes scanned and can modify the value and return a
@ -132,7 +139,7 @@ LxmlLinkExtractor
if m:
return m.group(1)
:type process_value: callable
:type process_value: collections.abc.Callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
@ -141,8 +148,16 @@ LxmlLinkExtractor
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
:type strip: bool
.. automethod:: extract_links
Link
----
.. module:: scrapy.link
:synopsis: Link from link extractors
.. autoclass:: Link
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

57
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -56,6 +56,8 @@ this:
error will be logged and the file won't be present in the ``files`` field.
.. _images-pipeline:
Using the Images Pipeline
=========================
@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
JPEG/RGB format, so you need to install this library in order to use it.
`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
PIL.
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
.. _Python Imaging Library: http://www.pythonware.com/products/pil/
.. _topics-media-pipeline-enabling:
@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
the passive connection mode by default. To use the active connection mode instead,
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
.. _media-pipelines-s3:
Amazon S3 storage
-----------------
.. setting:: FILES_STORE_S3_ACL
.. setting:: IMAGES_STORE_S3_ACL
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
bucket. Scrapy will automatically upload the files to the bucket.
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
@ -187,8 +188,9 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
@ -412,15 +415,16 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -436,12 +440,18 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@ -544,15 +554,16 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -568,12 +579,18 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,

76
docs/topics/request-response.rst
View File

@ -33,7 +33,7 @@ Request objects
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: string
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
@ -42,10 +42,10 @@ Request objects
:meth:`~scrapy.spiders.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
:type callback: callable
:type callback: collections.abc.Callable
:param method: the HTTP method of this request. Defaults to ``'GET'``.
:type method: string
:type method: str
:param meta: the initial values for the :attr:`Request.meta` attribute. If
given, the dict passed in this parameter will be shallow copied.
@ -61,6 +61,12 @@ Request objects
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
``None`` is passed as value, the HTTP header will not be sent at all.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
:type headers: dict
:param cookies: the request cookies. These can be sent in two forms.
@ -102,12 +108,18 @@ Request objects
)
For more info see :ref:`cookies-mw`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
:type cookies: dict or list
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to bytes (if given as a string).
:type encoding: string
:type encoding: str
:param priority: the priority of this request (defaults to ``0``).
The priority is used by the scheduler to define the order used to process
@ -119,7 +131,7 @@ Request objects
the scheduler. This is used when you want to perform an identical
request multiple times, to ignore the duplicates filter. Use it with
care, or you will get into crawling loops. Default to ``False``.
:type dont_filter: boolean
:type dont_filter: bool
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
@ -131,7 +143,7 @@ Request objects
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: callable
:type errback: collections.abc.Callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
@ -159,7 +171,7 @@ Request objects
.. attribute:: Request.body
A str that contains the request body.
The request body as bytes.
This attribute is read-only. To change the body of a Request use
:meth:`replace`.
@ -485,7 +497,7 @@ fields with form data from :class:`Response` objects.
:param formdata: is a dictionary (or iterable of (key, value) tuples)
containing HTML Form data which will be url-encoded and assigned to the
body of the request.
:type formdata: dict or iterable of tuples
:type formdata: dict or collections.abc.Iterable
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
@ -517,20 +529,20 @@ fields with form data from :class:`Response` objects.
:type response: :class:`Response` object
:param formname: if given, the form with name attribute set to this value will be used.
:type formname: string
:type formname: str
:param formid: if given, the form with id attribute set to this value will be used.
:type formid: string
:type formid: str
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
:type formxpath: str
:param formcss: if given, the first form that matches the css selector will be used.
:type formcss: string
:type formcss: str
:param formnumber: the number of form to use, when the response contains
multiple forms. The first one (and also the default) is ``0``.
:type formnumber: integer
:type formnumber: int
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
@ -548,23 +560,11 @@ fields with form data from :class:`Response` objects.
:param dont_click: If True, the form data will be submitted without
clicking in any element.
:type dont_click: boolean
:type dont_click: bool
The other parameters of this class method are passed directly to the
:class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
.. versionadded:: 0.17
The ``formxpath`` parameter.
.. versionadded:: 1.1.0
The ``formcss`` parameter.
.. versionadded:: 1.1.0
The ``formid`` parameter.
Request usage examples
----------------------
@ -636,7 +636,7 @@ dealing with JSON requests.
if :attr:`Request.body` argument is provided this parameter will be ignored.
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
:type data: object
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
@ -663,16 +663,16 @@ Response objects
downloaded (by the Downloader) and fed to the Spiders for processing.
:param url: the URL of this response
:type url: string
:type url: str
:param status: the HTTP status of the response. Defaults to ``200``.
:type status: integer
:type status: int
:param headers: the headers of this response. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. To access the decoded text as str you can use
:param body: the response body. To access the decoded text as a string, use
``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
@ -720,10 +720,10 @@ Response objects
.. attribute:: Response.body
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the string version use
:attr:`TextResponse.text` (only available in :class:`TextResponse`
and subclasses).
The response body as bytes.
If you want the body as a string, use :attr:`TextResponse.text` (only
available in :class:`TextResponse` and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@ -843,10 +843,10 @@ TextResponse objects
:param encoding: is a string which contains the encoding to use for this
response. If you create a :class:`TextResponse` object with a string as
body, it will be encoded using this encoding (remember the body attribute
is always a bytes object). If ``encoding`` is ``None`` (default value), the
encoding will be looked up in the response headers and body instead.
:type encoding: string
body, it will be converted to bytes encoded using this encoding. If
*encoding* is ``None`` (default), the encoding will be looked up in the
response headers and body instead.
:type encoding: str
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:

7
docs/topics/selectors.rst
View File

@ -328,8 +328,9 @@ too. Here's an example:
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
... href_xpath = link.xpath('@href').get()
... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
... print(" properties: %s" % (props.getall()))
... print(f" properties: {props.getall()}")
... print("")
current scope: ['http://schema.org/Product']

63
docs/topics/settings.rst
View File

@ -98,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Import paths and classes
========================
.. versionadded:: 2.4.0
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
- As a string containing the import path of that object
- As the object itself
For example::
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
}
.. note:: Passing non-callable objects is not supported.
How to access settings
======================
@ -110,7 +136,7 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com']
def parse(self, response):
print("Existing settings: %s" % self.settings.attributes.keys())
print(f"Existing settings: {self.settings.attributes.keys()}")
.. note::
The ``settings`` attribute is set in the base Spider class after the spider
@ -216,6 +242,32 @@ Default: ``None``
The name of the region associated with the AWS client.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
------------------
Default: ``None``
Import path of a given ``asyncio`` event loop class.
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
event loop will be used.
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
.. caution:: Please be aware that, when using a non-default event loop
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
:func:`asyncio.set_event_loop`, which will set the specified event loop
as the current loop for the current OS thread.
.. setting:: BOT_NAME
BOT_NAME
@ -306,6 +358,11 @@ Default::
The default headers used for Scrapy HTTP Requests. They're populated in the
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
.. setting:: DEPTH_LIMIT
DEPTH_LIMIT
@ -1030,8 +1087,6 @@ See :ref:`topics-extensions-ref-memusage`.
MEMUSAGE_CHECK_INTERVAL_SECONDS
-------------------------------
.. versionadded:: 1.1
Default: ``60.0``
Scope: ``scrapy.extensions.memusage``
@ -1312,8 +1367,6 @@ The class that will be used for loading spiders, which must implement the
SPIDER_LOADER_WARN_ONLY
-----------------------
.. versionadded:: 1.3.3
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,

5
docs/topics/signals.rst
View File

@ -423,6 +423,11 @@ response_received
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: The ``request`` argument might not contain the original request that
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
the :class:`~scrapy.http.Response` object and sets a specific ``request``
attribute.
response_downloaded
~~~~~~~~~~~~~~~~~~~

6
docs/topics/spider-middleware.rst
View File

@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
.. method:: process_start_requests(start_requests, spider)
.. versionadded:: 0.15
This method is called with the start requests of the spider, and works
similarly to the :meth:`process_spider_output` method, except that it
doesn't have a response associated and must return only requests (not
@ -341,8 +339,6 @@ RefererMiddleware settings
REFERER_ENABLED
^^^^^^^^^^^^^^^
.. versionadded:: 0.15
Default: ``True``
Whether to enable referer middleware.
@ -352,8 +348,6 @@ Whether to enable referer middleware.
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy

4
docs/topics/spiders.rst
View File

@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = ['http://www.example.com/categories/%s' % category]
self.start_urls = [f'http://www.example.com/categories/{category}']
# ...
The default `__init__` method will take any spider arguments
@ -292,7 +292,7 @@ The above example can also be written as follows::
name = 'myspider'
def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.

22
docs/versioning.rst
View File

@ -1,7 +1,7 @@
.. _versioning:
============================
Versioning and API Stability
Versioning and API stability
============================
Versioning
@ -34,7 +34,7 @@ For example:
production)
API Stability
API stability
=============
API stability was one of the major goals for the *1.0* release.
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
same way.
.. _deprecation-policy:
Deprecation policy
==================
We aim to maintain support for deprecated Scrapy features for at least 1 year.
For example, if a feature is deprecated in a Scrapy version released on
June 15th 2020, that feature should continue to work in versions released on
June 14th 2021 or before that.
Any new Scrapy release after a year *may* remove support for that deprecated
feature.
All deprecated features removed in a Scrapy release are explicitly mentioned in
the :ref:`release notes <news>`.
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases

2
extras/qps-bench-server.py
View File

@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args:
latency = float(request.args['latency'][0])

4
extras/qpsclient.py
View File

@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += '?latency={0}'.format(self.latency)
url += f'?latency={self.latency}'
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else:
urls = [url]

2
pylintrc
View File

@ -68,6 +68,7 @@ disable=abstract-method,
pointless-statement,
pointless-string-statement,
protected-access,
raise-missing-from,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
@ -75,6 +76,7 @@ disable=abstract-method,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,

2
pytest.ini
View File

@ -1,4 +1,5 @@
[pytest]
xfail_strict = true
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
@ -40,4 +41,3 @@ flake8-ignore =
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

2
scrapy/VERSION
View File

@ -1 +1 @@
2.3.0
2.4.1

4
scrapy/__init__.py
View File

@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 5, 2):
print("Scrapy %s requires Python 3.5.2" % __version__)
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
sys.exit(1)

16
scrapy/cmdline.py
View File

@ -44,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
else:
raise Exception("Invalid entry point %s" % entry_point.name)
raise Exception(f"Invalid entry point {entry_point.name}")
return cmds
@ -67,11 +67,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
@ -81,7 +81,7 @@ def _print_commands(settings, inproject):
print("Available commands:")
cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()):
print(" %-13s %s" % (cmdname, cmdclass.short_desc()))
print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject:
print()
print(" [ more ] More commands available when run from project directory")
@ -91,7 +91,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
@ -133,7 +133,7 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
@ -155,7 +155,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)

17
scrapy/commands/__init__.py
View File

@ -61,7 +61,7 @@ class ScrapyCommand:
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
if opts.output or opts.overwrite_output:
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

10
scrapy/commands/check.py
View File

@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else ""
writeln(self.separator2)
writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
writeln()
infos = []
@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED")
failed, errored = map(len, (self.failures, self.errors))
if failed:
infos.append("failures=%d" % failed)
infos.append(f"failures={failed}")
if errored:
infos.append("errors=%d" % errored)
infos.append(f"errors={errored}")
else:
write("OK")
if infos:
writeln(" (%s)" % (", ".join(infos),))
writeln(f" ({', '.join(infos)})")
else:
write("\n")
@ -85,7 +85,7 @@ class Command(ScrapyCommand):
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
print(f' * {method}')
else:
start = time.time()
self.crawler_process.start()

4
scrapy/commands/edit.py
View File

@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err("Spider not found: %s" % args[0])
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
self.exitcode = os.system('%s "%s"' % (editor, sfile))
self.exitcode = os.system(f'{editor} "{sfile}"')

60
scrapy/commands/genspider.py
View File

@ -66,31 +66,25 @@ class Command(ScrapyCommand):
print("Cannot create a spider with the same name as your project")
return
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider already exists and not --force then halt
if not opts.force:
print("Spider %r already exists in module:" % name)
print(" %s" % spidercls.__module__)
return
if not opts.force and self._spider_exists(name):
return
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
if opts.edit:
self.exitcode = os.system('scrapy edit "%s"' % name)
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
'classname': f'{capitalized_module}Spider'
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -98,26 +92,54 @@ class Command(ScrapyCommand):
else:
spiders_module = None
spiders_dir = "."
spider_file = "%s.py" % join(spiders_dir, module)
spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, '%s.tmpl' % template)
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
return template_file
print("Unable to find template: %s\n" % template)
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(" %s" % splitext(filename)[0])
print(f" {splitext(filename)[0]}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
return True
return False
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider with same name exists
print(f"Spider {name!r} already exists in module:")
print(f" {spidercls.__module__}")
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
return True
return False
@property
def templates_dir(self):

4
scrapy/commands/parse.py
View File

@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:

10
scrapy/commands/runspider.py
View File

@ -11,8 +11,8 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
raise ValueError("Not a Python source file: %s" % abspath)
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand):
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
raise UsageError("File not found: %s\n" % filename)
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError("Unable to load %r: %s\n" % (filename, e))
raise UsageError(f"Unable to load {filename!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError("No spider found in file: %s\n" % filename)
raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs)

12
scrapy/commands/startproject.py
View File

@ -52,7 +52,7 @@ class Command(ScrapyCommand):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
print(f'Error: Module {project_name!r} already exists')
else:
return True
return False
@ -100,7 +100,7 @@ class Command(ScrapyCommand):
if exists(join(project_dir, 'scrapy.cfg')):
self.exitcode = 1
print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
return
if not self._is_valid_name(project_name):
@ -113,11 +113,11 @@ class Command(ScrapyCommand):
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
print("You can start your first spider with:")
print(" cd %s" % project_dir)
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property

5
scrapy/commands/version.py
View File

@ -23,8 +23,7 @@ class Command(ScrapyCommand):
if opts.verbose:
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)
patt = "%-{}s : %s".format(width)
for name, version in versions:
print(patt % (name, version))
print(f"{name:<{width}} : {version}")
else:
print("Scrapy %s" % scrapy.__version__)
print(f"Scrapy {scrapy.__version__}")

8
scrapy/contracts/__init__.py
View File

@ -112,8 +112,8 @@ class Contract:
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name)
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
@ -172,8 +172,8 @@ def _create_testcase(method, desc):
class ContractTestCase(TestCase):
def __str__(_self):
return "[%s] %s (%s)" % (spider, method.__name__, desc)
return f"[{spider}] {method.__name__} ({desc})"
name = '%s_%s' % (spider, method.__name__)
name = f'{spider}_{method.__name__}'
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

12
scrapy/contracts/default.py
View File

@ -60,8 +60,7 @@ class ReturnsContract(Contract):
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
)
self.obj_name = self.args[0] or None
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
@ -88,10 +87,9 @@ class ReturnsContract(Contract):
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
expected = f'{self.min_bound}..{self.max_bound}'
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
class ScrapesContract(Contract):
@ -106,5 +104,5 @@ class ScrapesContract(Contract):
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
missing_str = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str)
missing_fields = ", ".join(missing)
raise ContractFail(f"Missing fields: {missing_fields}")

16
scrapy/core/downloader/__init__.py
View File

@ -41,17 +41,17 @@ class Slot:
def __repr__(self):
cls_name = self.__class__.__name__
return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % (
cls_name, self.concurrency, self.delay, self.randomize_delay)
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
def __str__(self):
return (
"<downloader.Slot concurrency=%r delay=%0.2f randomize_delay=%r "
"len(active)=%d len(queue)=%d len(transferring)=%d lastseen=%s>" % (
self.concurrency, self.delay, self.randomize_delay,
len(self.active), len(self.queue), len(self.transferring),
datetime.fromtimestamp(self.lastseen).isoformat()
)
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
)

3
scrapy/core/downloader/handlers/__init__.py
View File

@ -71,8 +71,7 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported("Unsupported URL scheme '%s': %s" %
(scheme, self._notconfigured[scheme]))
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
@defer.inlineCallbacks

38
scrapy/core/downloader/handlers/http11.py
View File

@ -60,11 +60,11 @@ class HTTP11DownloadHandler:
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
msg = f"""
'{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \
argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \
`tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them."""
warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
else:
extra = rcvd_bytes[:32]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
self._host, self._port, extra)))
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
)
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@ -371,7 +372,7 @@ class ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
@ -394,13 +395,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
error_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
logger.warning(warning_msg, warning_args)
logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
raise defer.CancelledError(error_msg % error_args)
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@ -523,11 +525,11 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()

59
scrapy/core/downloader/handlers/s3.py
View File

@ -2,41 +2,20 @@ from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
def _get_boto_connection():
from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, method, bucket, key, headers, *args, **kwargs):
return headers
class _v20_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, http_request, *args, **kwargs):
http_request.authorize(connection=self)
return http_request.headers
try:
import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
_S3Connection = _v20_S3Connection
return _S3Connection
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
@ -51,23 +30,15 @@ class S3DownloadHandler:
self.anon = kw.get('anon')
self._signer = None
if is_botocore():
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
_http_handler = create_instance(
objcls=httpdownloadhandler,
@ -85,14 +56,14 @@ class S3DownloadHandler:
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
headers=request.headers.to_unicode_dict(),
data=request.body)
self._signer.add_auth(awsrequest)

15
scrapy/core/downloader/middleware.py
View File

@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
f"Middleware {method.__self__.__class__.__name__}"
".process_request must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
return response
@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__self__.__class__.__name__}"
".process_response must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
return response
@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__self__.__class__.__name__}"
".process_exception must return None, Response or "
f"Request, got {type(response)}"
)
if response:
return response

27
scrapy/core/downloader/tls.py
View File

@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@ -28,13 +27,6 @@ openssl_methods = {
}
if twisted_version < (17, 0, 0):
from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
else:
def set_tlsext_host_name(connection, hostNameBytes):
connection.set_tlsext_host_name(hostNameBytes)
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
@ -52,21 +44,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL.SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
else:
logger.debug('SSL connection to %s using cipher %s',
self._hostnameASCII,
connection.get_cipher_name(),
)
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),

98
scrapy/core/downloader/webclient.py
View File

@ -1,9 +1,9 @@
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet import defer, reactor
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
@ -88,22 +88,38 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds."
% (self.factory.url, self.factory.timeout)))
defer.TimeoutError(f"Getting {self.factory.url} took longer "
f"than {self.factory.timeout} seconds."))
class ScrapyHTTPClientFactory(HTTPClientFactory):
"""Scrapy implementation of the HTTPClientFactory overwriting the
setUrl method to make use of our Url object that cache the parse
result.
"""
# This class used to inherit from Twisted’s
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overriden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
@ -138,21 +154,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
elif self.method == b'POST':
self.headers['Content-Length'] = 0
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
def __repr__(self):
return f"<{self.__class__.__name__}: {self.url}>"
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def _cancelTimeout(self, result, timeoutCall):
if timeoutCall.active():
timeoutCall.cancel()
return result
def buildProtocol(self, addr):
p = ClientFactory.buildProtocol(self, addr)
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
def gotStatus(self, version, status, message):
"""
Set the status of the request on us.
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}
"""
self.version, self.status, self.message = version, status, message
def page(self, page):
if self.waiting:
self.waiting = 0
self.deferred.callback(page)
def noPage(self, reason):
if self.waiting:
self.waiting = 0
self.deferred.errback(reason)
def clientConnectionFailed(self, _, reason):
"""
When a connection attempt fails, the request cannot be issued. If no
result has yet been provided to the result Deferred, provide the
connection failure reason as an error result.
"""
if self.waiting:
self.waiting = 0
# If the connection attempt failed, there is nothing more to
# disconnect, so just fire that Deferred now.
self._disconnectedDeferred.callback(None)
self.deferred.errback(reason)

25
scrapy/core/engine.py
View File

@ -171,8 +171,8 @@ class ExecutionEngine:
def _handle_downloader_output(self, response, request, spider):
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
"Incorrect type: expected Request, Response or Failure, got "
f"{type(response)}: {response!r}"
)
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
@ -214,7 +214,7 @@ class ExecutionEngine:
def crawl(self, request, spider):
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}")
self.schedule(request, spider)
self.slot.nextcall.schedule()
@ -239,16 +239,21 @@ class ExecutionEngine:
def _on_success(response):
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
"Incorrect type: expected Response or Request, got "
f"{type(response)}: {response!r}"
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
if response.request is None:
response.request = request
logkws = self.logformatter.crawled(response.request, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
self.signals.send_catch_log(signals.response_received,
response=response, request=request, spider=spider)
self.signals.send_catch_log(
signal=signals.response_received,
response=response,
request=response.request,
spider=spider,
)
return response
def _on_complete(_):
@ -263,7 +268,7 @@ class ExecutionEngine:
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)

64
scrapy/core/scraper.py
View File

@ -12,7 +12,7 @@ from scrapy import signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
@ -120,40 +120,40 @@ class Scraper:
response, request, deferred = slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
def _scrape(self, response, request, spider):
"""Handle the downloaded response or failure through the spider
callback/errback"""
if not isinstance(response, (Response, Failure)):
raise TypeError(
"Incorrect type: expected Response or Failure, got %s: %r"
% (type(response), response)
)
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
def _scrape(self, result, request, spider):
"""
Handle the downloaded response or failure through the spider callback/errback
"""
if not isinstance(result, (Response, Failure)):
raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}")
dfd = self._scrape2(result, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, result, spider)
dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
def _scrape2(self, request_result, request, spider):
"""Handle the different cases of request's result been a Response or a
Failure"""
if not isinstance(request_result, Failure):
return self.spidermw.scrape_response(
self.call_spider, request_result, request, spider)
else:
dfd = self.call_spider(request_result, request, spider)
return dfd.addErrback(
self._log_download_errors, request_result, request, spider)
def _scrape2(self, result, request, spider):
"""
Handle the different cases of request's result been a Response or a Failure
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
else: # result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
callback = request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
warn_on_generator_with_return_value(spider, request.errback)
dfd.addCallbacks(callback=callback,
errback=request.errback,
callbackKeywords=request.cb_kwargs)
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallback(callback, **result.request.cb_kwargs)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
@ -173,7 +173,7 @@ class Scraper:
spider=spider
)
self.crawler.stats.inc_value(
"spider_exceptions/%s" % _failure.value.__class__.__name__,
f"spider_exceptions/{_failure.value.__class__.__name__}",
spider=spider
)

20
scrapy/core/spidermw.py
View File

@ -19,10 +19,7 @@ def _isiterable(possible_iterator):
def _fname(f):
return "{}.{}".format(
f.__self__.__class__.__name__,
f.__func__.__name__
)
return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
class SpiderMiddlewareManager(MiddlewareManager):
@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
try:
result = method(response=response, spider=spider)
if result is not None:
msg = "Middleware {} must return None or raise an exception, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return None "
f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput:
raise
except Exception:
@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
msg = "Middleware {} must return None or an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return None "
f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure
def process_spider_output(result, start_index=0):
@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = _evaluate_iterable(result, method_index + 1, recovered)
else:
msg = "Middleware {} must return an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
msg = (f"Middleware {_fname(method)} must return an "
f"iterable, got {type(result)}")
raise _InvalidOutput(msg)
return MutableChain(result, recovered)

8
scrapy/crawler.py
View File

@ -180,9 +180,9 @@ class CrawlerRunner:
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
:class:`~scrapy.spiders.Spider` subclass or string
:param list args: arguments to initialize the spider
:param args: arguments to initialize the spider
:param dict kwargs: keyword arguments to initialize the spider
:param kwargs: keyword arguments to initialize the spider
"""
if isinstance(crawler_or_spidercls, Spider):
raise ValueError(
@ -307,7 +307,7 @@ class CrawlerProcess(CrawlerRunner):
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
from twisted.internet import reactor
@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner):
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"])
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
super()._handle_twisted_reactor()

55
scrapy/downloadermiddlewares/cookies.py
View File

@ -54,8 +54,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
msg = "Sending cookies to: {}\n{}".format(request, cookies)
cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
msg = f"Sending cookies to: {request}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _debug_set_cookie(self, response, spider):
@ -63,8 +63,8 @@ class CookiesMiddleware:
cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
msg = "Received cookies from: {}\n{}".format(response, cookies)
cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
msg = f"Received cookies from: {response}\n{cookies}"
logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie, request):
@ -74,7 +74,7 @@ class CookiesMiddleware:
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
if not cookie.get(key):
if cookie.get(key) is None:
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
@ -90,42 +90,21 @@ class CookiesMiddleware:
request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
for key, value in decoded.items(): # path, domain
cookie_str += "; {}={}".format(key.capitalize(), value)
cookie_str += f"; {key.capitalize()}={value}"
return cookie_str
def _get_request_cookies(self, jar, request):
"""
Extract cookies from a Request. Values from the `Request.cookies` attribute
take precedence over values from the `Cookie` request header.
Extract cookies from the Request.cookies attribute
"""
def get_cookies_from_header(jar, request):
cookie_header = request.headers.get("Cookie")
if not cookie_header:
return []
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
cookie_list_unicode = []
for cookie_bytes in cookie_gen_bytes:
try:
cookie_unicode = cookie_bytes.decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie_bytes)
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
cookie_list_unicode.append(cookie_unicode)
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
return jar.make_cookies(response, request)
def get_cookies_from_attribute(jar, request):
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
response = Response(request.url, headers={"Set-Cookie": formatted})
return jar.make_cookies(response, request)
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
response = Response(request.url, headers={"Set-Cookie": formatted})
return jar.make_cookies(response, request)

41
scrapy/downloadermiddlewares/httpcache.py
View File

@ -1,4 +1,5 @@
from email.utils import formatdate
from typing import Optional, Type, TypeVar
from twisted.internet import defer
from twisted.internet.error import (
@ -13,10 +14,19 @@ from twisted.internet.error import (
from twisted.web.client import ResponseFailed
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
@ -24,7 +34,7 @@ class HttpCacheMiddleware:
ConnectionLost, TCPTimedOutError, ResponseFailed,
IOError)
def __init__(self, settings, stats):
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
@ -33,26 +43,26 @@ class HttpCacheMiddleware:
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
def spider_opened(self, spider: Spider) -> None:
self.storage.open_spider(spider)
def spider_closed(self, spider):
def spider_closed(self, spider: Spider) -> None:
self.storage.close_spider(spider)
def process_request(self, request, spider):
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
if request.meta.get('dont_cache', False):
return
return None
# Skip uncacheable requests
if not self.policy.should_cache_request(request):
request.meta['_dont_cache'] = True # flag as uncacheable
return
return None
# Look for cached response and check if expired
cachedresponse = self.storage.retrieve_response(spider, request)
@ -61,7 +71,7 @@ class HttpCacheMiddleware:
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
raise IgnoreRequest("Ignored request not in cache: %s" % request)
return # first time request
return None # first time request
# Return cached response only if not expired
cachedresponse.flags.append('cached')
@ -73,7 +83,9 @@ class HttpCacheMiddleware:
# process_response hook
request.meta['cached_response'] = cachedresponse
def process_response(self, request, response, spider):
return None
def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
if request.meta.get('dont_cache', False):
return response
@ -85,7 +97,7 @@ class HttpCacheMiddleware:
# RFC2616 requires origin server to set Date header,
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
if 'Date' not in response.headers:
response.headers['Date'] = formatdate(usegmt=1)
response.headers['Date'] = formatdate(usegmt=True)
# Do not validate first-hand responses
cachedresponse = request.meta.pop('cached_response', None)
@ -102,13 +114,18 @@ class HttpCacheMiddleware:
self._cache_response(spider, response, request, cachedresponse)
return response
def process_exception(self, request, exception, spider):
def process_exception(
self, request: Request, exception: Exception, spider: Spider
) -> Optional[Response]:
cachedresponse = request.meta.pop('cached_response', None)
if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
self.stats.inc_value('httpcache/errorrecovery', spider=spider)
return cachedresponse
return None
def _cache_response(self, spider, response, request, cachedresponse):
def _cache_response(
self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
) -> None:
if self.policy.should_cache_response(response, request):
self.stats.inc_value('httpcache/store', spider=spider)
self.storage.store_response(spider, request, response)

12
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,3 +1,4 @@
import io
import zlib
from scrapy.utils.gz import gunzip
@ -14,6 +15,12 @@ try:
except ImportError:
pass
try:
import zstandard
ACCEPTED_ENCODINGS.append(b'zstd')
except ImportError:
pass
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
@ -67,4 +74,9 @@ class HttpCompressionMiddleware:
body = zlib.decompress(body, -15)
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
body = brotli.decompress(body)
if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
# Using its streaming API since its simple API could handle only cases
# where there is content size data embedded in the frame
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
body = reader.read()
return body

9
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -13,7 +13,12 @@ class HttpProxyMiddleware:
self.auth_encoding = auth_encoding
self.proxies = {}
for type_, url in getproxies().items():
self.proxies[type_] = self._get_proxy(url, type_)
try:
self.proxies[type_] = self._get_proxy(url, type_)
# some values such as '/var/run/docker.sock' can't be parsed
# by _parse_proxy and as such should be skipped
except ValueError:
continue
@classmethod
def from_crawler(cls, crawler):
@ -24,7 +29,7 @@ class HttpProxyMiddleware:
def _basic_auth_header(self, username, password):
user_pass = to_bytes(
'%s:%s' % (unquote(username), unquote(password)),
f'{unquote(username)}:{unquote(password)}',
encoding=self.auth_encoding)
return base64.b64encode(user_pass)

2
scrapy/downloadermiddlewares/retry.py
View File

@ -88,7 +88,7 @@ class RetryMiddleware:
reason = global_object_name(reason.__class__)
stats.inc_value('retry/count')
stats.inc_value('retry/reason_count/%s' % reason)
stats.inc_value(f'retry/reason_count/{reason}')
return retryreq
else:
stats.inc_value('retry/max_reached')

6
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -61,7 +61,7 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers:
self._parsers[netloc] = Deferred()
robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
@ -94,7 +94,7 @@ class RobotsTxtMiddleware:
def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}')
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
@ -102,7 +102,7 @@ class RobotsTxtMiddleware:
def _robots_error(self, failure, netloc):
if failure.type is not IgnoreRequest:
key = 'robotstxt/exception_count/{}'.format(failure.type)
key = f'robotstxt/exception_count/{failure.type}'
self.crawler.stats.inc_value(key)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = None

6
scrapy/downloadermiddlewares/stats.py
View File

@ -17,13 +17,13 @@ class DownloaderStats:
def process_request(self, request, spider):
self.stats.inc_value('downloader/request_count', spider=spider)
self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
reqlen = len(request_httprepr(request))
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response_httprepr(response))
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response
@ -31,4 +31,4 @@ class DownloaderStats:
def process_exception(self, request, exception, spider):
ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)

7
scrapy/exporters.py
View File

@ -40,7 +40,7 @@ class BaseItemExporter:
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None)
if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
def export_item(self, item):
raise NotImplementedError
@ -208,7 +208,7 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
@ -218,7 +218,8 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
errors=errors,
)
self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True

2
scrapy/extensions/corestats.py
View File

@ -43,4 +43,4 @@ class CoreStats:
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
self.stats.inc_value('item_dropped_count', spider=spider)
self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)

2
scrapy/extensions/debug.py
View File

@ -48,7 +48,7 @@ class StackTraceDump:
for id_, frame in sys._current_frames().items():
name = id2name.get(id_, '')
dump = ''.join(traceback.format_stack(frame))
dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
dumps += f"# Thread: {name}({id_})\n{dump}\n"
return dumps

217
scrapy/extensions/feedexport.py
View File

@ -19,22 +19,39 @@ from zope.interface import implementer, Interface
from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
if 'feed_options' in argument_names:
kwargs['feed_options'] = feed_options
else:
warnings.warn(
"{} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy."
.format(builder.__qualname__),
category=ScrapyDeprecationWarning
)
return builder(*preargs, uri, *args, **kwargs)
class IFeedStorage(Interface):
"""Interface that all Feed Storages must implement"""
def __init__(uri):
"""Initialize the storage with the parameters given in the URI"""
def __init__(uri, *, feed_options=None):
"""Initialize the storage with the parameters given in the URI and the
feed-specific options (see :setting:`FEEDS`)"""
def open(spider):
"""Open the storage for the given spider. It must return a file-like
@ -64,10 +81,15 @@ class BlockingFeedStorage:
@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(self, uri, _stdout=None):
def __init__(self, uri, _stdout=None, *, feed_options=None):
if not _stdout:
_stdout = sys.stdout.buffer
self._stdout = _stdout
if feed_options and feed_options.get('overwrite', False) is True:
logger.warning('Standard output (stdout) storage does not support '
'overwriting. To suppress this warning, remove the '
'overwrite option from your FEEDS setting, or set '
'it to False.')
def open(self, spider):
return self._stdout
@ -79,14 +101,16 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri):
def __init__(self, uri, *, feed_options=None):
self.path = file_uri_to_path(uri)
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, 'ab')
return open(self.path, self.write_mode)
def store(self, file):
file.close()
@ -94,64 +118,44 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None):
# BEGIN Backward compatibility for initialising without keys (and
# without using from_crawler)
no_defaults = access_key is None and secret_key is None
if no_defaults:
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
warnings.warn(
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
"without AWS keys is deprecated. Please supply credentials or "
"use the `from_crawler()` constructor.",
category=ScrapyDeprecationWarning,
stacklevel=2
)
access_key = settings['AWS_ACCESS_KEY_ID']
secret_key = settings['AWS_SECRET_ACCESS_KEY']
# END Backward compatibility
def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
feed_options=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
else:
import boto
self.connect_s3 = boto.connect_s3
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
'option from your FEEDS setting or set it to True.')
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
uri=uri,
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
feed_options=feed_options,
)
def _store_in_thread(self, file):
file.seek(0)
if self.is_botocore:
kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file,
**kwargs)
else:
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
kwargs = {'policy': self.acl} if self.acl else {}
key.set_contents_from_file(file, **kwargs)
key.close()
kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file,
**kwargs)
file.close()
class GCSFeedStorage(BlockingFeedStorage):
@ -182,27 +186,31 @@ class GCSFeedStorage(BlockingFeedStorage):
class FTPFeedStorage(BlockingFeedStorage):
def __init__(self, uri, use_active_mode=False):
def __init__(self, uri, use_active_mode=False, *, feed_options=None):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.username = u.username
self.password = unquote(u.password)
self.password = unquote(u.password or '')
self.path = u.path
self.use_active_mode = use_active_mode
self.overwrite = not feed_options or feed_options.get('overwrite', True)
@classmethod
def from_crawler(cls, crawler, uri):
return cls(
uri=uri,
use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
def from_crawler(cls, crawler, uri, *, feed_options=None):
return build_storage(
cls,
uri,
crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
feed_options=feed_options,
)
def _store_in_thread(self, file):
ftp_store_file(
path=self.path, file=file, host=self.host,
port=self.port, username=self.username,
password=self.password, use_active_mode=self.use_active_mode
password=self.password, use_active_mode=self.use_active_mode,
overwrite=self.overwrite,
)
@ -259,32 +267,32 @@ class FeedExporter:
category=ScrapyDeprecationWarning, stacklevel=2,
)
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed in self.settings.getdict('FEEDS').items():
for uri, feed_options in self.settings.getdict('FEEDS').items():
uri = str(uri) # handle pathlib.Path objects
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
for uri, feed_options in self.feeds.items():
if not self._storage_supported(uri, feed_options):
raise NotConfigured
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed['format']):
if not self._exporter_supported(feed_options['format']):
raise NotConfigured
def open_spider(self, spider):
for uri, feed in self.feeds.items():
uri_params = self._get_uri_params(spider, feed['uri_params'])
for uri, feed_options in self.feeds.items():
uri_params = self._get_uri_params(spider, feed_options['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
feed=feed,
feed_options=feed_options,
spider=spider,
uri_template=uri,
))
@ -311,44 +319,53 @@ class FeedExporter:
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
)
return d
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
:param feed_options: dict with parameters of feed
:param spider: user spider
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
"""
storage = self._get_storage(uri)
storage = self._get_storage(uri, feed_options)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
format=feed['format'],
fields_to_export=feed['fields'],
encoding=feed['encoding'],
indent=feed['indent'],
format=feed_options['format'],
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
**feed_options['item_export_kwargs'],
)
slot = _FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
format=feed['format'],
store_empty=feed['store_empty'],
format=feed_options['format'],
store_empty=feed_options['store_empty'],
batch_id=batch_id,
uri_template=uri_template,
)
@ -372,7 +389,7 @@ class FeedExporter:
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
feed=self.feeds[slot.uri_template],
feed_options=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
))
@ -411,11 +428,11 @@ class FeedExporter:
return False
return True
def _storage_supported(self, uri):
def _storage_supported(self, uri, feed_options):
scheme = urlparse(uri).scheme
if scheme in self.storages:
try:
self._get_storage(uri)
self._get_storage(uri, feed_options)
return True
except NotConfigured as e:
logger.error("Disabled feed storage scheme: %(scheme)s. "
@ -433,8 +450,30 @@ class FeedExporter:
def _get_exporter(self, file, format, *args, **kwargs):
return self._get_instance(self.exporters[format], file, *args, **kwargs)
def _get_storage(self, uri):
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
def _get_storage(self, uri, feed_options):
"""Fork of create_instance specific to feed storage classes
It supports not passing the *feed_options* parameters to classes that
do not support it, and issuing a deprecation warning instead.
"""
feedcls = self.storages[urlparse(uri).scheme]
crawler = getattr(self, 'crawler', None)
def build_instance(builder, *preargs):
return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
if crawler and hasattr(feedcls, 'from_crawler'):
instance = build_instance(feedcls.from_crawler, crawler)
method_name = 'from_crawler'
elif hasattr(feedcls, 'from_settings'):
instance = build_instance(feedcls.from_settings, self.settings)
method_name = 'from_settings'
else:
instance = build_instance(feedcls)
method_name = '__new__'
if instance is None:
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
return instance
def _get_uri_params(self, spider, uri_params, slot=None):
params = {}

10
scrapy/extensions/httpcache.py
View File

@ -223,7 +223,7 @@ class DbmCacheStorage:
self.db = None
def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
@ -251,13 +251,13 @@ class DbmCacheStorage:
'headers': dict(response.headers),
'body': response.body,
}
self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
self.db['%s_time' % key] = str(time())
self.db[f'{key}_data'] = pickle.dumps(data, protocol=4)
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
key = self._request_key(request)
db = self.db
tkey = '%s_time' % key
tkey = f'{key}_time'
if tkey not in db:
return # not found
@ -265,7 +265,7 @@ class DbmCacheStorage:
if 0 < self.expiration_secs < time() - float(ts):
return # expired
return pickle.loads(db['%s_data' % key])
return pickle.loads(db[f'{key}_data'])
def _request_key(self, request):
return request_fingerprint(request)

2
scrapy/extensions/memdebug.py
View File

@ -30,4 +30,4 @@ class MemoryDebugger:
for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)

14
scrapy/extensions/memusage.py
View File

@ -82,8 +82,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
"%s terminated: memory usage exceeded %dM at %s"
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}M at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1)
@ -105,8 +105,8 @@ class MemoryUsage:
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
"%s warning: memory usage reached %dM at %s"
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}M at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1)
@ -115,9 +115,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"

10
scrapy/extensions/statsmailer.py
View File

@ -24,11 +24,11 @@ class StatsMailer:
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_closed(self, spider):
spider_stats = self.stats.get_stats(spider)
body = "Global stats\n\n"
body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items())
body += "\n\n%s stats\n\n" % spider.name
body += "\n".join("%-50s : %s" % i for i in spider_stats.items())
return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body)
body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
body += f"\n\n{spider.name} stats\n\n"
body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items())
return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body)

2
scrapy/http/common.py
View File

@ -1,6 +1,6 @@
def obsolete_setter(setter, attrname):
def newsetter(self, value):
c = self.__class__.__name__
msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c)
msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
raise AttributeError(msg)
return newsetter

2
scrapy/http/headers.py
View File

@ -33,7 +33,7 @@ class Headers(CaselessDict):
elif isinstance(x, int):
return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key):
try:

18
scrapy/http/request/__init__.py
View File

@ -25,13 +25,13 @@ class Request(object_ref):
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
raise TypeError("Request priority not an integer: %r" % priority)
raise TypeError(f"Request priority not an integer: {priority!r}")
self.priority = priority
if callback is not None and not callable(callback):
raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
raise TypeError(f'callback must be a callable, got {type(callback).__name__}')
if errback is not None and not callable(errback):
raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
raise TypeError(f'errback must be a callable, got {type(errback).__name__}')
self.callback = callback
self.errback = errback
@ -60,13 +60,17 @@ class Request(object_ref):
def _set_url(self, url):
if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}')
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
if ('://' not in self._url) and (not self._url.startswith('data:')):
raise ValueError('Missing scheme in request url: %s' % self._url)
if (
'://' not in self._url
and not self._url.startswith('about:')
and not self._url.startswith('data:')
):
raise ValueError(f'Missing scheme in request url: {self._url}')
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -86,7 +90,7 @@ class Request(object_ref):
return self._encoding
def __str__(self):
return "<%s %s>" % (self.method, self.url)
return f"<{self.method} {self.url}>"
__repr__ = __str__

19
scrapy/http/request/form.py
View File

@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath):
base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)
raise ValueError(f"No <form> element found in {response}")
if formname is not None:
f = root.xpath('//form[@name="%s"]' % formname)
f = root.xpath(f'//form[@name="{formname}"]')
if f:
return f[0]
if formid is not None:
f = root.xpath('//form[@id="%s"]' % formid)
f = root.xpath(f'//form[@id="{formid}"]')
if f:
return f[0]
@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
el = el.getparent()
if el is None:
break
raise ValueError('No <form> element found with %s' % formxpath)
raise ValueError(f'No <form> element found with {formxpath}')
# If we get here, it means that either formname was None
# or invalid
@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
try:
form = forms[formnumber]
except IndexError:
raise IndexError("Form number %d not found in %s" %
(formnumber, response))
raise IndexError(f"Form number {formnumber} not found in {response}")
else:
return form
@ -205,12 +204,12 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
elif len(el) > 1:
raise ValueError("Multiple elements found (%r) matching the criteria "
"in clickdata: %r" % (el, clickdata))
raise ValueError(f"Multiple elements found ({el!r}) matching the "
f"criteria in clickdata: {clickdata!r}")
else:
raise ValueError('No clickable element matching clickdata: %r' % (clickdata,))
raise ValueError(f'No clickable element matching clickdata: {clickdata!r}')

6
scrapy/http/response/__init__.py
View File

@ -55,8 +55,8 @@ class Response(object_ref):
if isinstance(url, str):
self._url = url
else:
raise TypeError('%s url must be str, got %s:' %
(type(self).__name__, type(url).__name__))
raise TypeError(f'{type(self).__name__} url must be str, '
f'got {type(url).__name__}')
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@ -77,7 +77,7 @@ class Response(object_ref):
body = property(_get_body, obsolete_setter(_set_body, 'body'))
def __str__(self):
return "<%d %s>" % (self.status, self.url)
return f"<{self.status} {self.url}>"
__repr__ = __str__

15
scrapy/http/response/text.py
View File

@ -47,8 +47,8 @@ class TextResponse(Response):
self._body = b'' # used by encoding detection
if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
raise TypeError('Cannot convert unicode body - '
f'{type(self).__name__} has no encoding')
self._body = body.encode(self._encoding)
else:
super()._set_body(body)
@ -92,7 +92,7 @@ class TextResponse(Response):
# _body_inferred_encoding is called
benc = self.encoding
if self._cached_ubody is None:
charset = 'charset=%s' % benc
charset = f'charset={benc}'
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
@ -255,12 +255,11 @@ def _url_from_selector(sel):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise _InvalidSelector("Unsupported selector: %s" % sel)
raise _InvalidSelector(f"Unsupported selector: {sel}")
if sel.root.tag not in ('a', 'link'):
raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
sel.root.tag)
raise _InvalidSelector("Only <a> and <link> elements are supported; "
f"got <{sel.root.tag}>")
href = sel.root.get('href')
if href is None:
raise _InvalidSelector("<%s> element has no href attribute: %s" %
(sel.root.tag, sel))
raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}")
return strip_html5_whitespace(href)

6
scrapy/item.py
View File

@ -96,19 +96,19 @@ class DictItem(MutableMapping, BaseItem):
if key in self.fields:
self._values[key] = value
else:
raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
raise KeyError(f"{self.__class__.__name__} does not support field: {key}")
def __delitem__(self, key):
del self._values[key]
def __getattr__(self, name):
if name in self.fields:
raise AttributeError("Use item[%r] to get field value" % name)
raise AttributeError(f"Use item[{name!r}] to get field value")
raise AttributeError(name)
def __setattr__(self, name, value):
if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
super().__setattr__(name, value)
def __len__(self):

23
scrapy/link.py
View File

@ -7,14 +7,29 @@ its documentation in: docs/topics/link-extractors.rst
class Link:
"""Link objects represent an extracted link by the LinkExtractor."""
"""Link objects represent an extracted link by the LinkExtractor.
Using the anchor tag sample below to illustrate the parameters::
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
:param url: the absolute url being linked to in the anchor tag.
From the sample, this is ``https://example.com/nofollow.html``.
:param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
:param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
:param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
of the anchor tag.
"""
__slots__ = ['url', 'text', 'fragment', 'nofollow']
def __init__(self, url, text='', fragment='', nofollow=False):
if not isinstance(url, str):
got = url.__class__.__name__
raise TypeError("Link urls must be str objects, got %s" % got)
raise TypeError(f"Link urls must be str objects, got {got}")
self.url = url
self.text = text
self.fragment = fragment
@ -33,6 +48,6 @@ class Link:
def __repr__(self):
return (
'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
% (self.url, self.text, self.fragment, self.nofollow)
f'Link(url={self.url!r}, text={self.text!r}, '
f'fragment={self.fragment!r}, nofollow={self.nofollow!r})'
)

4
scrapy/logformatter.py
View File

@ -54,8 +54,8 @@ class LogFormatter:
def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''
response_flags = ' %s' % str(response.flags) if response.flags else ''
request_flags = f' {str(request.flags)}' if request.flags else ''
response_flags = f' {str(response.flags)}' if response.flags else ''
return {
'level': logging.DEBUG,
'msg': CRAWLEDMSG,

136
scrapy/pipelines/files.py
View File

@ -11,7 +11,6 @@ import os
import time
from collections import defaultdict
from contextlib import suppress
from email.utils import mktime_tz, parsedate_tz
from ftplib import FTP
from io import BytesIO
from urllib.parse import urlparse
@ -23,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
@ -91,86 +90,54 @@ class S3FilesStore:
}
def __init__(self, uri):
self.is_botocore = is_botocore()
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY
)
else:
from boto.s3.connection import S3Connection
self.S3Connection = S3Connection
if not is_botocore_available():
raise NotConfigured('missing botocore library')
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY
)
if not uri.startswith("s3://"):
raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri)
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split('/', 1)
def stat_file(self, path, info):
def _onsuccess(boto_key):
if self.is_botocore:
checksum = boto_key['ETag'].strip('"')
last_modified = boto_key['LastModified']
modified_stamp = time.mktime(last_modified.timetuple())
else:
checksum = boto_key.etag.strip('"')
last_modified = boto_key.last_modified
modified_tuple = parsedate_tz(last_modified)
modified_stamp = int(mktime_tz(modified_tuple))
checksum = boto_key['ETag'].strip('"')
last_modified = boto_key['LastModified']
modified_stamp = time.mktime(last_modified.timetuple())
return {'checksum': checksum, 'last_modified': modified_stamp}
return self._get_boto_key(path).addCallback(_onsuccess)
def _get_boto_bucket(self):
# disable ssl (is_secure=False) because of this python bug:
# https://bugs.python.org/issue5103
c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
return c.get_bucket(self.bucket, validate=False)
def _get_boto_key(self, path):
key_name = '%s%s' % (self.prefix, path)
if self.is_botocore:
return threads.deferToThread(
self.s3_client.head_object,
Bucket=self.bucket,
Key=key_name)
else:
b = self._get_boto_bucket()
return threads.deferToThread(b.get_key, key_name)
key_name = f'{self.prefix}{path}'
return threads.deferToThread(
self.s3_client.head_object,
Bucket=self.bucket,
Key=key_name)
def persist_file(self, path, buf, info, meta=None, headers=None):
"""Upload file to S3 storage"""
key_name = '%s%s' % (self.prefix, path)
key_name = f'{self.prefix}{path}'
buf.seek(0)
if self.is_botocore:
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
h.update(headers)
return threads.deferToThread(
k.set_contents_from_string, buf.getvalue(),
headers=h, policy=self.POLICY)
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
def _headers_to_botocore_kwargs(self, headers):
""" Convert headers to botocore keyword agruments.
@ -208,8 +175,7 @@ class S3FilesStore:
try:
kwarg = mapping[key]
except KeyError:
raise TypeError(
'Header "%s" is not supported by botocore' % key)
raise TypeError(f'Header "{key}" is not supported by botocore')
else:
extra[kwarg] = value
return extra
@ -283,7 +249,7 @@ class FTPFilesStore:
def __init__(self, uri):
if not uri.startswith("ftp://"):
raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri)
raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
u = urlparse(uri)
self.port = u.port
self.host = u.hostname
@ -293,7 +259,7 @@ class FTPFilesStore:
self.basedir = u.path.rstrip('/')
def persist_file(self, path, buf, info, meta=None, headers=None):
path = '%s/%s' % (self.basedir, path)
path = f'{self.basedir}/{path}'
return threads.deferToThread(
ftp_store_file, path=path, file=buf,
host=self.host, port=self.port, username=self.username,
@ -308,10 +274,10 @@ class FTPFilesStore:
ftp.login(self.username, self.password)
if self.USE_ACTIVE_MODE:
ftp.set_pasv(False)
file_path = "%s/%s" % (self.basedir, path)
last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip())
file_path = f"{self.basedir}/{path}"
last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
m = hashlib.md5()
ftp.retrbinary('RETR %s' % file_path, m.update)
ftp.retrbinary(f'RETR {file_path}', m.update)
return {'last_modified': last_modified, 'checksum': m.hexdigest()}
# The file doesn't exist
except Exception:
@ -409,7 +375,7 @@ class FilesPipeline(MediaPipeline):
store_cls = self.STORE_SCHEMES[scheme]
return store_cls(uri)
def media_to_download(self, request, info):
def media_to_download(self, request, info, *, item=None):
def _onsuccess(result):
if not result:
return # returning None force download
@ -436,7 +402,7 @@ class FilesPipeline(MediaPipeline):
checksum = result.get('checksum', None)
return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
path = self.file_path(request, info=info)
path = self.file_path(request, info=info, item=item)
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
dfd.addCallbacks(_onsuccess, lambda _: None)
dfd.addErrback(
@ -460,7 +426,7 @@ class FilesPipeline(MediaPipeline):
raise FileException
def media_downloaded(self, response, request, info):
def media_downloaded(self, response, request, info, *, item=None):
referer = referer_str(request)
if response.status != 200:
@ -492,8 +458,8 @@ class FilesPipeline(MediaPipeline):
self.inc_stats(info.spider, status)
try:
path = self.file_path(request, response=response, info=info)
checksum = self.file_downloaded(response, request, info)
path = self.file_path(request, response=response, info=info, item=item)
checksum = self.file_downloaded(response, request, info, item=item)
except FileException as exc:
logger.warning(
'File (error): Error processing file from %(request)s '
@ -515,15 +481,15 @@ class FilesPipeline(MediaPipeline):
def inc_stats(self, spider, status):
spider.crawler.stats.inc_value('file_count', spider=spider)
spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider)
# Overridable Interface
def get_media_requests(self, item, info):
urls = ItemAdapter(item).get(self.files_urls_field, [])
return [Request(u) for u in urls]
def file_downloaded(self, response, request, info):
path = self.file_path(request, response=response, info=info)
def file_downloaded(self, response, request, info, *, item=None):
path = self.file_path(request, response=response, info=info, item=item)
buf = BytesIO(response.body)
checksum = md5sum(buf)
buf.seek(0)
@ -535,7 +501,7 @@ class FilesPipeline(MediaPipeline):
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1]
# Handles empty and wild extensions by trying to guess the
@ -545,4 +511,4 @@ class FilesPipeline(MediaPipeline):
media_type = mimetypes.guess_type(request.url)[0]
if media_type:
media_ext = mimetypes.guess_extension(media_type)
return 'full/%s%s' % (media_guid, media_ext)
return f'full/{media_guid}{media_ext}'

23
scrapy/pipelines/images.py
View File

@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline):
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)
def file_downloaded(self, response, request, info):
return self.image_downloaded(response, request, info)
def file_downloaded(self, response, request, info, *, item=None):
return self.image_downloaded(response, request, info, item=item)
def image_downloaded(self, response, request, info):
def image_downloaded(self, response, request, info, *, item=None):
checksum = None
for path, image, buf in self.get_images(response, request, info):
for path, image, buf in self.get_images(response, request, info, item=item):
if checksum is None:
buf.seek(0)
checksum = md5sum(buf)
@ -119,14 +119,15 @@ class ImagesPipeline(FilesPipeline):
headers={'Content-Type': 'image/jpeg'})
return checksum
def get_images(self, response, request, info):
path = self.file_path(request, response=response, info=info)
def get_images(self, response, request, info, *, item=None):
path = self.file_path(request, response=response, info=info, item=item)
orig_image = Image.open(BytesIO(response.body))
width, height = orig_image.size
if width < self.min_width or height < self.min_height:
raise ImageException("Image too small (%dx%d < %dx%d)" %
(width, height, self.min_width, self.min_height))
raise ImageException("Image too small "
f"({width}x{height} < "
f"{self.min_width}x{self.min_height})")
image, buf = self.convert_image(orig_image)
yield path, image, buf
@ -166,10 +167,10 @@ class ImagesPipeline(FilesPipeline):
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'full/%s.jpg' % (image_guid)
return f'full/{image_guid}.jpg'
def thumb_path(self, request, thumb_id, response=None, info=None):
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'

70
scrapy/pipelines/media.py
View File

@ -1,12 +1,16 @@
import functools
import logging
from collections import defaultdict
from inspect import signature
from warnings import warn
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.log import failure_to_exc_info
@ -27,6 +31,7 @@ class MediaPipeline:
def __init__(self, download_func=None, settings=None):
self.download_func = download_func
self._expects_item = {}
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
@ -38,6 +43,9 @@ class MediaPipeline:
)
self._handle_statuses(self.allow_redirects)
# Check if deprecated methods are being used and make them compatible
self._make_compatible()
def _handle_statuses(self, allow_redirects):
self.handle_httpstatus_list = None
if allow_redirects:
@ -53,7 +61,7 @@ class MediaPipeline:
'MYPIPE_IMAGES'
"""
class_name = self.__class__.__name__
formatted_key = "{}_{}".format(class_name.upper(), key)
formatted_key = f"{class_name.upper()}_{key}"
if (
not base_class_name
or class_name == base_class_name
@ -77,11 +85,11 @@ class MediaPipeline:
def process_item(self, item, spider):
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info) for r in requests]
dlist = [self._process_request(r, info, item) for r in requests]
dfd = DeferredList(dlist, consumeErrors=1)
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info):
def _process_request(self, request, info, item):
fp = request_fingerprint(request)
cb = request.callback or (lambda _: _)
eb = request.errback
@ -102,34 +110,72 @@ class MediaPipeline:
# Download request checking media_to_download hook output first
info.downloading.add(fp)
dfd = mustbe_deferred(self.media_to_download, request, info)
dfd.addCallback(self._check_media_to_download, request, info)
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
dfd.addCallback(self._check_media_to_download, request, info, item=item)
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
dfd.addErrback(lambda f: logger.error(
f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
)
return dfd.addBoth(lambda _: wad) # it must return wad at last
def _make_compatible(self):
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
methods = [
"file_path", "media_to_download", "media_downloaded",
"file_downloaded", "image_downloaded", "get_images"
]
for method_name in methods:
method = getattr(self, method_name, None)
if callable(method):
setattr(self, method_name, self._compatible(method))
def _compatible(self, func):
"""Wrapper for overridable methods to allow backwards compatibility"""
self._check_signature(func)
@functools.wraps(func)
def wrapper(*args, **kwargs):
if self._expects_item[func.__name__]:
return func(*args, **kwargs)
kwargs.pop('item', None)
return func(*args, **kwargs)
return wrapper
def _check_signature(self, func):
sig = signature(func)
self._expects_item[func.__name__] = True
if 'item' not in sig.parameters:
old_params = str(sig)[1:-1]
new_params = old_params + ", *, item=None"
warn(f'{func.__name__}(self, {old_params}) is deprecated, '
f'please use {func.__name__}(self, {new_params})',
ScrapyDeprecationWarning, stacklevel=2)
self._expects_item[func.__name__] = False
def _modify_media_request(self, request):
if self.handle_httpstatus_list:
request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
else:
request.meta['handle_httpstatus_all'] = True
def _check_media_to_download(self, result, request, info):
def _check_media_to_download(self, result, request, info, item):
if result is not None:
return result
if self.download_func:
# this ugly code was left only to support tests. TODO: remove
dfd = mustbe_deferred(self.download_func, request, info.spider)
dfd.addCallbacks(
callback=self.media_downloaded, callbackArgs=(request, info),
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
errback=self.media_failed, errbackArgs=(request, info))
else:
self._modify_media_request(request)
dfd = self.crawler.engine.download(request, info.spider)
dfd.addCallbacks(
callback=self.media_downloaded, callbackArgs=(request, info),
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
errback=self.media_failed, errbackArgs=(request, info))
return dfd
@ -171,7 +217,7 @@ class MediaPipeline:
defer_result(result).chainDeferred(wad)
# Overridable Interface
def media_to_download(self, request, info):
def media_to_download(self, request, info, *, item=None):
"""Check request before starting download"""
pass
@ -179,7 +225,7 @@ class MediaPipeline:
"""Returns the media requests to download"""
pass
def media_downloaded(self, response, request, info):
def media_downloaded(self, response, request, info, *, item=None):
"""Handler for success downloads"""
return response
@ -199,3 +245,7 @@ class MediaPipeline:
extra={'spider': info.spider}
)
return item
def file_path(self, request, response=None, info=None, *, item=None):
"""Returns the path where downloaded media should be stored"""
pass

9
scrapy/pqueues.py
View File

@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue:
def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
% (self.__class__,))
raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP')
if slot_startprios and not isinstance(slot_startprios, dict):
raise ValueError("DownloaderAwarePriorityQueue accepts "
"``slot_startprios`` as a dict; %r instance "
"``slot_startprios`` as a dict; "
f"{slot_startprios.__class__!r} instance "
"is passed. Most likely, it means the state is"
"created by an incompatible priority queue. "
"Only a crawl started with the same priority "
"queue class can be resumed." %
slot_startprios.__class__)
"queue class can be resumed.")
self._downloader_interface = DownloaderInterface(crawler)
self.downstream_queue_cls = downstream_queue_cls

67
scrapy/resolver.py
View File

@ -1,6 +1,6 @@
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
@ -50,6 +50,36 @@ class CachingThreadedResolver(ThreadedResolver):
return result
@implementer(IHostResolution)
class HostResolution:
def __init__(self, name):
self.name = name
def cancel(self):
raise NotImplementedError()
@provider(IResolutionReceiver)
class _CachingResolutionReceiver:
def __init__(self, resolutionReceiver, hostName):
self.resolutionReceiver = resolutionReceiver
self.hostName = hostName
self.addresses = []
def resolutionBegan(self, resolution):
self.resolutionReceiver.resolutionBegan(resolution)
self.resolution = resolution
def addressResolved(self, address):
self.resolutionReceiver.addressResolved(address)
self.addresses.append(address)
def resolutionComplete(self):
self.resolutionReceiver.resolutionComplete()
if self.addresses:
dnscache[self.hostName] = self.addresses
@implementer(IHostnameResolver)
class CachingHostnameResolver:
"""
@ -73,33 +103,22 @@ class CachingHostnameResolver:
def install_on_reactor(self):
self.reactor.installNameResolver(self)
def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
addressTypes=None, transportSemantics='TCP'):
@provider(IResolutionReceiver)
class CachingResolutionReceiver(resolutionReceiver):
def resolutionBegan(self, resolution):
super().resolutionBegan(resolution)
self.resolution = resolution
self.resolved = False
def addressResolved(self, address):
super().addressResolved(address)
self.resolved = True
def resolutionComplete(self):
super().resolutionComplete()
if self.resolved:
dnscache[hostName] = self.resolution
def resolveHostName(
self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP"
):
try:
return dnscache[hostName]
addresses = dnscache[hostName]
except KeyError:
return self.original_resolver.resolveHostName(
CachingResolutionReceiver(),
_CachingResolutionReceiver(resolutionReceiver, hostName),
hostName,
portNumber,
addressTypes,
transportSemantics
transportSemantics,
)
else:
resolutionReceiver.resolutionBegan(HostResolution(hostName))
for addr in addresses:
resolutionReceiver.addressResolved(addr)
resolutionReceiver.resolutionComplete()
return resolutionReceiver

Some files were not shown because too many files have changed in this diff Show More