RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into http-proxy-schema

This commit is contained in:
ajaymittur28 2021-06-14 21:18:49 +05:30
parent 065b9b1170 9f81de2a50
commit 857e0e225a
322 changed files with 14015 additions and 6043 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.2.0
current_version = 2.5.0
commit = True
tag = True
tag_name = {new_version}

19
.flake8 Normal file
View File

@ -0,0 +1,19 @@
[flake8]
max-line-length = 119
ignore = W503
exclude =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
tests/sample_data/** binary

40
.github/workflows/checks.yml vendored Normal file
View File

@ -0,0 +1,40 @@
name: Checks
on: [push, pull_request]
jobs:
checks:
runs-on: ubuntu-18.04
strategy:
matrix:
include:
- python-version: 3.9
env:
TOXENV: security
- python-version: 3.9
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: pylint
- python-version: 3.9
env:
TOXENV: typing
- python-version: 3.8 # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run check
env: ${{ matrix.env }}
run: |
pip install -U tox
tox

31
.github/workflows/publish.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: Publish
on: [push]
jobs:
publish:
runs-on: ubuntu-18.04
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*

25
.github/workflows/tests-macos.yml vendored Normal file
View File

@ -0,0 +1,25 @@
name: macOS
on: [push, pull_request]
jobs:
tests:
runs-on: macos-10.15
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
run: |
pip install -U tox
tox -e py
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

74
.github/workflows/tests-ubuntu.yml vendored Normal file
View File

@ -0,0 +1,74 @@
name: Ubuntu
on: [push, pull_request]
jobs:
tests:
runs-on: ubuntu-18.04
strategy:
matrix:
include:
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: pypy3
env:
TOXENV: pypy3
PYPY_VERSION: 3.6-v7.3.1
# pinned deps
- python-version: 3.6.12
env:
TOXENV: pinned
- python-version: 3.6.12
env:
TOXENV: asyncio-pinned
- python-version: pypy3
env:
TOXENV: pypy3-pinned
PYPY_VERSION: 3.6-v7.2.0
# extras
# extra-deps includes reppy, which does not support Python 3.9
# https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: extra-deps
- python-version: 3.9
env:
TOXENV: asyncio
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
run: |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
pip install -U tox
tox
- name: Upload coverage report
run: bash <(curl -s https://codecov.io/bash)

36
.github/workflows/tests-windows.yml vendored Normal file
View File

@ -0,0 +1,36 @@
name: Windows
on: [push, pull_request]
jobs:
tests:
runs-on: windows-latest
strategy:
matrix:
include:
- python-version: 3.6
env:
TOXENV: windows-pinned
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
# https://twistedmatrix.com/trac/ticket/9990
#- python-version: 3.9
#env:
#TOXENV: py
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox

4
.gitignore vendored
View File

@ -14,8 +14,12 @@ htmlcov/
.coverage
.pytest_cache/
.coverage.*
coverage.*
test-output.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
/tests/keys/localhost.key
# Windows
Thumbs.db

6
.readthedocs.yml
View File

@ -3,10 +3,14 @@ formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
build:
image: latest
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
version: 3.8 # Keep in sync with .github/workflows/checks.yml
install:
- requirements: docs/requirements.txt
- path: .

75
.travis.yml
View File

@ -1,75 +0,0 @@
language: python
dist: xenial
branches:
only:
- master
- /^\d\.\d+$/
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- env: TOXENV=security
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pylint
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=typing
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=pinned
python: 3.5.2
- env: TOXENV=asyncio
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
- env: TOXENV=py
python: 3.5
- env: TOXENV=asyncio
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
- env: TOXENV=py
python: 3.6
- env: TOXENV=py
python: 3.7
- env: TOXENV=py PYPI_RELEASE_JOB=true
python: 3.8
dist: bionic
- env: TOXENV=extra-deps
python: 3.8
dist: bionic
- env: TOXENV=asyncio
python: 3.8
dist: bionic
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
- pip install -U tox twine wheel codecov
script: tox
after_success:
- codecov
notifications:
irc:
use_notice: true
skip_join: true
channels:
- irc.freenode.org#scrapy
cache:
directories:
- $HOME/.cache/pip
deploy:
provider: pypi
distributions: "sdist bdist_wheel"
user: scrapy
password:
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
on:
tags: true
repo: scrapy/scrapy
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

4
AUTHORS
View File

@ -1,8 +1,8 @@
Scrapy was brought to life by Shane Evans while hacking a scraping framework
prototype for Mydeco (mydeco.com). It soon became maintained, extended and
improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
bootstrap the project. In mid-2011, Scrapinghub became the new official
maintainer.
bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
official maintainer.
Here is the list of the primary authors & contributors:

5
CODE_OF_CONDUCT.md
View File

@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at opensource@scrapinghub.com. All
reported by contacting the project team at opensource@zyte.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq

25
README.rst
View File

@ -10,9 +10,17 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
:target: https://travis-ci.org/scrapy/scrapy
:alt: Build Status
.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
:alt: Windows
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
:target: https://pypi.python.org/pypi/Scrapy
@ -34,13 +42,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
.. _Zyte: https://www.zyte.com/
Check the Scrapy homepage at https://scrapy.org for more information,
including a list of features.
Requirements
============
* Python 3.5.2+
* Python 3.6+
* Works on Linux, Windows, macOS, BSD
Install
@ -81,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@scrapinghub.com.
Please report unacceptable behavior to opensource@zyte.com.
Companies using Scrapy
======================

25
appveyor.yml
View File

@ -1,25 +0,0 @@
platform: x86
version: '{branch}-{build}'
environment:
matrix:
- PYTHON: "C:\\Python36"
TOX_ENV: py36
branches:
only:
- master
- /d+\.\d+\.\d+[\w\-]*$/
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
- "pip install -U tox"
build: false
skip_tags: true
test_script:
- "tox -e %TOX_ENV%"
cache:
- '%LOCALAPPDATA%\pip\cache'

41
conftest.py
View File

@ -1,6 +1,11 @@
from pathlib import Path
import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys
def _py_files(folder):
@ -14,14 +19,21 @@ collect_ignore = [
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
with open('tests/ignores.txt') as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
*_py_files("scrapy/core/http2"),
)
)
@pytest.fixture()
@ -40,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items):
pass
def pytest_addoption(parser):
parser.addoption(
"--reactor",
default="default",
choices=["default", "asyncio"],
)
@pytest.fixture(scope='class')
def reactor_pytest(request):
if not request.cls:
@ -53,3 +73,12 @@ def reactor_pytest(request):
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
# Generate localhost certificate files, needed by some tests
generate_keys()

2
docs/Makefile
View File

@ -8,7 +8,7 @@ PYTHON = python
SPHINXOPTS =
PAPER =
SOURCES =
SHELL = /bin/bash
SHELL = /usr/bin/env bash
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
-D latex_elements.papersize=$(PAPER) \

2
docs/_ext/scrapydocs.py
View File

@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
# index entries for setting directives look like:
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
return False

10
docs/_static/custom.css vendored Normal file
View File

@ -0,0 +1,10 @@
/* Move lists closer to their introducing paragraph */
.rst-content .section ol p, .rst-content .section ul p {
margin-bottom: 0px;
}
.rst-content p + ol, .rst-content p + ul {
margin-top: -18px; /* Compensates margin-top: 24px of p */
}
.rst-content dl p + ol, .rst-content dl p + ul {
margin-top: -6px; /* Compensates margin-top: 12px of p */
}

7
docs/_templates/layout.html vendored
View File

@ -3,14 +3,9 @@
{% block footer %}
{{ super() }}
<script type="text/javascript">
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
analytics.page();
}}();
analytics.ready(function () {
ga('require', 'linker');
ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
ga('linker:autoLink', ['zyte.com']);
});
</script>
{% endblock %}

21
docs/conf.py
View File

@ -49,7 +49,7 @@ master_doc = 'index'
# General information about the project.
project = 'Scrapy'
copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
copyright = f'2008–{datetime.now().year}, Scrapy developers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
import sphinx_rtd_theme
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
@ -183,6 +182,10 @@ html_copy_source = True
# Output file base name for HTML help builder.
htmlhelp_basename = 'Scrapydoc'
html_css_files = [
'custom.css',
]
# Options for LaTeX output
# ------------------------
@ -283,7 +286,9 @@ coverage_ignore_pyobjects = [
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cryptography' : ('https://cryptography.io/en/latest/', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
@ -305,3 +310,15 @@ hoverxref_role_types = {
"ref": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
def setup(app):
app.connect('autodoc-skip-member', maybe_skip_member)
def maybe_skip_member(app, what, name, obj, skip, options):
if not skip:
# autodocs was generating a text "alias of" for the following members
# https://github.com/sphinx-doc/sphinx/issues/4422
return name in {'default_item_class', 'default_selector_class'}
return skip

18
docs/contributing.rst
View File

@ -108,6 +108,11 @@ Well-written patches should:
tox -e docs-coverage
* if you are removing deprecated code, first make sure that at least 1 year
(12 months) has passed since the release that introduced the deprecation.
See :ref:`deprecation-policy`.
.. _submitting-patches:
Submitting patches
@ -135,7 +140,7 @@ original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
not considered rude as long as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
@ -194,6 +199,17 @@ In any case, if something is covered in a docstring, use the
documentation instead of duplicating the docstring in files within the
``docs/`` directory.
Documentation updates that cover new or modified features must use Sphinx’s
:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use
``VERSION`` as version, we will replace it with the actual version right before
the corresponding release. When we release a new major or minor version of
Scrapy, we remove these directives if they are older than 3 years.
Documentation about deprecated features must be removed as those features are
deprecated, so that new readers do not run into it. New deprecations and
deprecation removals are documented in the :ref:`release notes <news>`.
Tests
=====

66
docs/faq.rst
View File

@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
.. _faq-python-versions:
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 3.5.2+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
Python 2 support was dropped in Scrapy 2.0.
.. note::
For Python 3 support on Windows, it is recommended to use
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
Did Scrapy "steal" X from Django?
---------------------------------
@ -159,6 +145,41 @@ How can I make Scrapy consume less memory?
See previous question.
How can I prevent memory errors due to many allowed domains?
------------------------------------------------------------
If you have a spider with a long list of
:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider
replacing the default
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
less memory. For example:
- If your domain names are similar enough, use your own regular expression
instead joining the strings in
:attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular
expression.
- If you can `meet the installation requirements`_, use pyre2_ instead of
Python’s re_ to compile your URL-filtering regular expression. See
:issue:`1908`.
See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
.. _pyre2: https://github.com/andreasvc/pyre2
.. _re: https://docs.python.org/library/re.html
.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
Can I use Basic HTTP Authentication in my spiders?
--------------------------------------------------
@ -250,15 +271,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
To dump into a JSON file::
scrapy crawl myspider -o items.json
scrapy crawl myspider -O items.json
To dump into a CSV file::
scrapy crawl myspider -o items.csv
scrapy crawl myspider -O items.csv
To dump into a XML file::
scrapy crawl myspider -o items.xml
scrapy crawl myspider -O items.xml
For more information see :ref:`topics-feed-exports`
@ -377,11 +398,12 @@ How can I cancel the download of a given response?
--------------------------------------------------
In some situations, it might be useful to stop the download of a certain response.
For instance, if you only need the first part of a large response and you would like
to save resources by avoiding the download of the whole body.
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
the :ref:`topics-stop-response-download` topic for additional information and examples.
For instance, sometimes you can determine whether or not you need the full contents
of a response by inspecting its headers or the first bytes of its body. In that case,
you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
or :class:`~scrapy.signals.headers_received` signals and raising a
:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
:ref:`topics-stop-response-download` topic for additional information and examples.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905

5
docs/index.rst
View File

@ -78,7 +78,6 @@ Basic concepts
topics/settings
topics/exceptions
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.
@ -228,6 +227,7 @@ Extending Scrapy
topics/extensions
topics/api
topics/signals
topics/scheduler
topics/exporters
@ -249,6 +249,9 @@ Extending Scrapy
:doc:`topics/signals`
See all available signals and how to work with them.
:doc:`topics/scheduler`
Understand the scheduler component.
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).

51
docs/intro/install.rst
View File

@ -4,12 +4,19 @@
Installation guide
==================
.. _faq-python-versions:
Supported Python versions
=========================
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
Installing Scrapy
=================
Scrapy runs on Python 3.5.2 or above under CPython (default Python
implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
and macOS.
@ -23,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
For more detailed and platform specifics instructions, as well as
troubleshooting information, read on.
@ -63,10 +70,9 @@ In case of any trouble related to these dependencies,
please refer to their respective installation instructions:
* `lxml installation`_
* `cryptography installation`_
* :doc:`cryptography installation <cryptography:installation>`
.. _lxml installation: https://lxml.de/installation.html
.. _cryptography installation: https://cryptography.io/en/latest/installation/
.. _intro-using-virtualenv:
@ -112,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
conda install -c conda-forge scrapy
To install Scrapy on Windows using ``pip``:
.. warning::
This installation method requires “Microsoft Visual C++” for installing some
Scrapy dependencies, which demands significantly more disk space than Anaconda.
#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
#. Run the Visual Studio Installer.
#. Under the Workloads section, select **C++ build tools**.
#. Check the installation details and make sure following packages are selected as optional components:
* **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
* **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0))
#. Install the Visual Studio Build Tools.
Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
.. _intro-install-ubuntu:
@ -205,8 +232,8 @@ PyPy
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
@ -259,10 +286,10 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _cryptography: https://cryptography.io/en/latest/
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
.. _conda-forge: https://conda-forge.org/

26
docs/intro/overview.rst
View File

@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
if next_page is not None:
yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.json
scrapy runspider quotes_spider.py -o quotes.jl
When this finishes you will have in the ``quotes.jl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
When this finishes you will have in the ``quotes.json`` file a list of the
quotes in JSON format, containing text and author, looking like this (reformatted
here for better readability)::
[{
"author": "Jane Austen",
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
},
{
"author": "Groucho Marx",
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
},
{
"author": "Steve Martin",
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
},
...]
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
{"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
...
What just happened?

23
docs/intro/tutorial.rst
View File

@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
self.log('Saved file %s' % filename)
self.log(f'Saved file {filename}')
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
@ -190,7 +190,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = 'quotes-%s.html' % page
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
@ -405,8 +405,6 @@ to get all of them:
from sys import version_info
.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
@ -464,16 +462,15 @@ Storing the scraped data
The simplest way to store the scraped data is by using :ref:`Feed exports
<topics-feed-exports>`, with the following command::
scrapy crawl quotes -o quotes.json
scrapy crawl quotes -O quotes.json
That will generate an ``quotes.json`` file containing all scraped items,
That will generate a ``quotes.json`` file containing all scraped items,
serialized in `JSON`_.
For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also use other formats, like `JSON Lines`_::
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
@ -704,7 +701,7 @@ Using spider arguments
You can provide command line arguments to your spiders by using the ``-a``
option when running them::
scrapy crawl quotes -o quotes-humor.json -a tag=humor
scrapy crawl quotes -O quotes-humor.json -a tag=humor
These arguments are passed to the Spider's ``__init__`` method and become
spider attributes by default.

657
docs/news.rst
View File

@ -3,6 +3,652 @@
Release notes
=============
.. _release-2.5.0:
Scrapy 2.5.0 (2021-04-06)
-------------------------
Highlights:
- Official Python 3.9 support
- Experimental :ref:`HTTP/2 support <http2>`
- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function
to retry requests from spider callbacks
- New :class:`~scrapy.signals.headers_received` signal that allows stopping
downloads early
- New :class:`Response.protocol <scrapy.http.Response.protocol>` attribute
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and
had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`.
(:issue:`4901`)
- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`)
Deprecations
~~~~~~~~~~~~
- The :mod:`scrapy.utils.py36` module is now deprecated in favor of
:mod:`scrapy.utils.asyncgen`. (:issue:`4900`)
New features
~~~~~~~~~~~~
- Experimental :ref:`HTTP/2 support <http2>` through a new download handler
that can be assigned to the ``https`` protocol in the
:setting:`DOWNLOAD_HANDLERS` setting.
(:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)
- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request`
function may be used from spider callbacks or middlewares to handle the
retrying of a request beyond the scenarios that
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports.
(:issue:`3590`, :issue:`3685`, :issue:`4902`)
- The new :class:`~scrapy.signals.headers_received` signal gives early access
to response headers and allows :ref:`stopping downloads
<topics-stop-response-download>`.
(:issue:`1772`, :issue:`4897`)
- The new :attr:`Response.protocol <scrapy.http.Response.protocol>`
attribute gives access to the string that identifies the protocol used to
download a response. (:issue:`4878`)
- :ref:`Stats <topics-stats>` now include the following entries that indicate
the number of successes and failures in storing
:ref:`feeds <topics-feed-exports>`::
feedexport/success_count/<storage type>
feedexport/failed_count/<storage type>
Where ``<storage type>`` is the feed storage backend class name, such as
:class:`~scrapy.extensions.feedexport.FileFeedStorage` or
:class:`~scrapy.extensions.feedexport.FTPFeedStorage`.
(:issue:`3947`, :issue:`4850`)
- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider
middleware now logs ignored URLs with ``INFO`` :ref:`logging level
<levels>` instead of ``DEBUG``, and it now includes the following entry
into :ref:`stats <topics-stats>` to keep track of the number of ignored
URLs::
urllength/request_ignored_count
(:issue:`5036`)
- The
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
downloader middleware now logs the number of decompressed responses and the
total count of resulting bytes::
httpcompression/response_bytes
httpcompression/response_count
(:issue:`4797`, :issue:`4799`)
Bug fixes
~~~~~~~~~
- Fixed installation on PyPy installing PyDispatcher in addition to
PyPyDispatcher, which could prevent Scrapy from working depending on which
package got imported. (:issue:`4710`, :issue:`4814`)
- When inspecting a callback to check if it is a generator that also returns
a value, an exception is no longer raised if the callback has a docstring
with lower indentation than the following code.
(:issue:`4477`, :issue:`4935`)
- The `Content-Length <https://tools.ietf.org/html/rfc2616#section-14.13>`_
header is no longer omitted from responses when using the default, HTTP/1.1
download handler (see :setting:`DOWNLOAD_HANDLERS`).
(:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`)
- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False``
now has the same effect as not setting it at all, instead of having the
same effect as setting it to ``True``.
(:issue:`3851`, :issue:`4694`)
Documentation
~~~~~~~~~~~~~
- Added instructions to :ref:`install Scrapy in Windows using pip
<intro-install-windows>`.
(:issue:`4715`, :issue:`4736`)
- Logging documentation now includes :ref:`additional ways to filter logs
<topics-logging-advanced-customization>`.
(:issue:`4216`, :issue:`4257`, :issue:`4965`)
- Covered how to deal with long lists of allowed domains in the :ref:`FAQ
<faq>`. (:issue:`2263`, :issue:`3667`)
- Covered scrapy-bench_ in :ref:`benchmarking`.
(:issue:`4996`, :issue:`5016`)
- Clarified that one :ref:`extension <topics-extensions>` instance is created
per crawler.
(:issue:`5014`)
- Fixed some errors in examples.
(:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`,
:issue:`5008`)
- Fixed some external links, typos, and so on.
(:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`,
:issue:`5063`)
- The :ref:`list of Request.meta keys <topics-request-meta>` is now sorted
alphabetically.
(:issue:`5061`, :issue:`5065`)
- Updated references to Scrapinghub, which is now called Zyte.
(:issue:`4973`, :issue:`5072`)
- Added a mention to contributors in the README. (:issue:`4956`)
- Reduced the top margin of lists. (:issue:`4974`)
Quality Assurance
~~~~~~~~~~~~~~~~~
- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`)
- Extended typing hints (:issue:`4895`)
- Fixed deprecated uses of the Twisted API.
(:issue:`4940`, :issue:`4950`, :issue:`5073`)
- Made our tests run with the new pip resolver.
(:issue:`4710`, :issue:`4814`)
- Added tests to ensure that :ref:`coroutine support <coroutine-support>`
is tested. (:issue:`4987`)
- Migrated from Travis CI to GitHub Actions. (:issue:`4924`)
- Fixed CI issues.
(:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`,
:issue:`5053`)
- Implemented code refactorings, style fixes and cleanups.
(:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`)
.. _release-2.4.1:
Scrapy 2.4.1 (2020-11-17)
-------------------------
- Fixed :ref:`feed exports <topics-feed-exports>` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`)
- Fixed the AsyncIO event loop handling, which could make code hang
(:issue:`4855`, :issue:`4872`)
- Fixed the IPv6-capable DNS resolver
:class:`~scrapy.resolver.CachingHostnameResolver` for download handlers
that call
:meth:`reactor.resolve <twisted.internet.interfaces.IReactorCore.resolve>`
(:issue:`4802`, :issue:`4803`)
- Fixed the output of the :command:`genspider` command showing placeholders
instead of the import path of the generated spider module (:issue:`4874`)
- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
:issue:`4876`)
.. _release-2.4.0:
Scrapy 2.4.0 (2020-10-11)
-------------------------
Highlights:
* Python 3.5 support has been dropped.
* The ``file_path`` method of :ref:`media pipelines <topics-media-pipeline>`
can now access the source :ref:`item <topics-items>`.
This allows you to set a download file path based on item data.
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
to define keyword parameters to pass to :ref:`item exporter classes
<topics-exporters>`
* You can now choose whether :ref:`feed exports <topics-feed-exports>`
overwrite or append to the output file.
For example, when using the :command:`crawl` or :command:`runspider`
commands, you can use the ``-O`` option instead of ``-o`` to overwrite the
output file.
* Zstd-compressed responses are now supported if zstandard_ is installed.
* In settings, where the import path of a class is required, it is now
possible to pass a class object instead.
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
* Python 3.6 or greater is now required; support for Python 3.5 has been
dropped
As a result:
- When using PyPy, PyPy 7.2.0 or greater :ref:`is now required
<faq-python-versions>`
- For Amazon S3 storage support in :ref:`feed exports
<topics-feed-storage-s3>` or :ref:`media pipelines
<media-pipelines-s3>`, botocore_ 1.4.87 or greater is now required
- To use the :ref:`images pipeline <images-pipeline>`, Pillow_ 4.0.0 or
greater is now required
(:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`,
:issue:`4764`)
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again
discards cookies defined in :attr:`Request.headers
<scrapy.http.Request.headers>`.
We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it
was reported that the current implementation could break existing code.
If you need to set cookies for a request, use the :class:`Request.cookies
<scrapy.http.Request>` parameter.
A future version of Scrapy will include a new, better implementation of the
reverted bug fix.
(:issue:`4717`, :issue:`4823`)
.. _2.4-deprecation-removals:
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the
values of ``access_key`` and ``secret_key`` from the running project
settings when they are not passed to its ``__init__`` method; you must
either pass those parameters to its ``__init__`` method or use
:class:`S3FeedStorage.from_crawler
<scrapy.extensions.feedexport.S3FeedStorage.from_crawler>`
(:issue:`4356`, :issue:`4411`, :issue:`4688`)
* :attr:`Rule.process_request <scrapy.spiders.crawl.Rule.process_request>`
no longer admits callables which expect a single ``request`` parameter,
rather than both ``request`` and ``response`` (:issue:`4818`)
Deprecations
~~~~~~~~~~~~
* In custom :ref:`media pipelines <topics-media-pipeline>`, signatures that
do not accept a keyword-only ``item`` parameter in any of the methods that
:ref:`now support this parameter <media-pipeline-item-parameter>` are now
deprecated (:issue:`4628`, :issue:`4686`)
* In custom :ref:`feed storage backend classes <topics-feed-storage>`,
``__init__`` method signatures that do not accept a keyword-only
``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`,
:issue:`4512`)
* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated
(:issue:`4684`, :issue:`4701`)
* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use
:func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`,
:issue:`4776`)
New features
~~~~~~~~~~~~
.. _media-pipeline-item-parameter:
* The following methods of :ref:`media pipelines <topics-media-pipeline>` now
accept an ``item`` keyword-only parameter containing the source
:ref:`item <topics-items>`:
- In :class:`scrapy.pipelines.files.FilesPipeline`:
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded`
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_path`
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download`
- In :class:`scrapy.pipelines.images.ImagesPipeline`:
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded`
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download`
(:issue:`4628`, :issue:`4686`)
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
to define keyword parameters to pass to :ref:`item exporter classes
<topics-exporters>` (:issue:`4606`, :issue:`4768`)
* :ref:`Feed exports <topics-feed-exports>` gained overwrite support:
* When using the :command:`crawl` or :command:`runspider` commands, you
can use the ``-O`` option instead of ``-o`` to overwrite the output
file
* You can use the ``overwrite`` key in the :setting:`FEEDS` setting to
configure whether to overwrite the output file (``True``) or append to
its content (``False``)
* The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage
backend classes <topics-feed-storage>` now receive a new keyword-only
parameter, ``feed_options``, which is a dictionary of :ref:`feed
options <feed-options>`
(:issue:`547`, :issue:`716`, :issue:`4512`)
* Zstd-compressed responses are now supported if zstandard_ is installed
(:issue:`4831`)
* In settings, where the import path of a class is required, it is now
possible to pass a class object instead (:issue:`3870`, :issue:`3873`).
This includes also settings where only part of its value is made of an
import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or
:setting:`DOWNLOAD_HANDLERS`.
* :ref:`Downloader middlewares <topics-downloader-middleware>` can now
override :class:`response.request <scrapy.http.Response.request>`.
If a :ref:`downloader middleware <topics-downloader-middleware>` returns
a :class:`~scrapy.http.Response` object from
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
or
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
with a custom :class:`~scrapy.http.Request` object assigned to
:class:`response.request <scrapy.http.Response.request>`:
- The response is handled by the callback of that custom
:class:`~scrapy.http.Request` object, instead of being handled by the
callback of the original :class:`~scrapy.http.Request` object
- That custom :class:`~scrapy.http.Request` object is now sent as the
``request`` argument to the :signal:`response_received` signal, instead
of the original :class:`~scrapy.http.Request` object
(:issue:`4529`, :issue:`4632`)
* When using the :ref:`FTP feed storage backend <topics-feed-storage-ftp>`:
- It is now possible to set the new ``overwrite`` :ref:`feed option
<feed-options>` to ``False`` to append to an existing file instead of
overwriting it
- The FTP password can now be omitted if it is not necessary
(:issue:`547`, :issue:`716`, :issue:`4512`)
* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now
supports an ``errors`` parameter to indicate how to handle encoding errors
(:issue:`4755`)
* When :ref:`using asyncio <using-asyncio>`, it is now possible to
:ref:`set a custom asyncio loop <using-custom-loops>` (:issue:`4306`,
:issue:`4414`)
* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are
spider methods that delegate on other callable (:issue:`4756`)
* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged
message is now a warning, instead of an error (:issue:`3874`,
:issue:`3886`, :issue:`4752`)
Bug fixes
~~~~~~~~~
* The :command:`genspider` command no longer overwrites existing files
unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`,
:issue:`4623`)
* Cookies with an empty value are no longer considered invalid cookies
(:issue:`4772`)
* The :command:`runspider` command now supports files with the ``.pyw`` file
extension (:issue:`4643`, :issue:`4646`)
* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
middleware now simply ignores unsupported proxy values (:issue:`3331`,
:issue:`4778`)
* Checks for generator callbacks with a ``return`` statement no longer warn
about ``return`` statements in nested functions (:issue:`4720`,
:issue:`4721`)
* The system file mode creation mask no longer affects the permissions of
files generated using the :command:`startproject` command (:issue:`4722`)
* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
(:issue:`861`, :issue:`4746`)
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
work when using a headless browser (:issue:`4835`)
Documentation
~~~~~~~~~~~~~
* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`,
:issue:`4724`)
* Improved the documentation of
:ref:`link extractors <topics-link-extractors>` with an usage example from
a spider callback and reference documentation for the
:class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`)
* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the
:class:`~scrapy.extensions.closespider.CloseSpider` extension
(:issue:`4836`)
* Removed references to Python 2’s ``unicode`` type (:issue:`4547`,
:issue:`4703`)
* We now have an :ref:`official deprecation policy <deprecation-policy>`
(:issue:`4705`)
* Our :ref:`documentation policies <documentation-policies>` now cover usage
of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged`
directives, and we have removed usages referencing Scrapy 1.4.0 and earlier
versions (:issue:`3971`, :issue:`4310`)
* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`,
:issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Extended typing hints (:issue:`4243`, :issue:`4691`)
* Added tests for the :command:`check` command (:issue:`4663`)
* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`)
* Improved Windows test coverage (:issue:`4723`)
* Switched to :ref:`formatted string literals <f-strings>` where possible
(:issue:`4307`, :issue:`4324`, :issue:`4672`)
* Modernized :func:`super` usage (:issue:`4707`)
* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`,
:issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`,
:issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`,
:issue:`4820`, :issue:`4822`, :issue:`4839`)
.. _release-2.3.0:
Scrapy 2.3.0 (2020-08-04)
-------------------------
Highlights:
* :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` as a storage backend
* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
output items in batches of up to the specified number of items.
It also serves as a workaround for :ref:`delayed file delivery
<delayed-file-delivery>`, which causes Scrapy to only start item delivery
after the crawl has finished when using certain storage backends
(:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
and now :ref:`GCS <topics-feed-storage-gcs>`).
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
allowing usage from outside Scrapy and a separate release schedule
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* Removed the following classes and their parent modules from
``scrapy.linkextractors``:
* ``htmlparser.HtmlParserLinkExtractor``
* ``regex.RegexLinkExtractor``
* ``sgml.BaseSgmlLinkExtractor``
* ``sgml.SgmlLinkExtractor``
Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead (:issue:`4356`, :issue:`4679`)
Deprecations
~~~~~~~~~~~~
* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
(:issue:`4683`)
New features
~~~~~~~~~~~~
* :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
(:issue:`4250`, :issue:`4434`)
* The :command:`parse` command now allows specifying an output file
(:issue:`4317`, :issue:`4377`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
``--data-raw`` (:issue:`4612`)
* A ``parse`` callback may now be used in built-in spider subclasses, such
as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
:issue:`781`, :issue:`4254` )
Bug fixes
~~~~~~~~~
* Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
:ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
<attrs-items>` (:issue:`4667`, :issue:`4668`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
method to ``POST`` when a request body is specified and no request method
is specified (:issue:`4612`)
* The processing of ANSI escape sequences in enabled in Windows 10.0.14393
and later, where it is required for colored output (:issue:`4393`,
:issue:`4403`)
Documentation
~~~~~~~~~~~~~
* Updated the `OpenSSL cipher list format`_ link in the documentation about
the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
* Simplified the code example in :ref:`topics-loaders-dataclass`
(:issue:`4652`)
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
Quality assurance
~~~~~~~~~~~~~~~~~
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
:issue:`4516`)
* Fixed a silenced error in some scheduler tests (:issue:`4644`,
:issue:`4645`)
* Renewed the localhost certificate used for SSL tests (:issue:`4650`)
* Removed cookie-handling code specific to Python 2 (:issue:`4682`)
* Stopped using Python 2 unicode literal syntax (:issue:`4704`)
* Stopped using a backlash for line continuation (:issue:`4673`)
* Removed unneeded entries from the MyPy exception list (:issue:`4690`)
* Automated tests now pass on Windows as part of our continuous integration
system (:issue:`4458`)
* Automated tests now pass on the latest PyPy version for supported Python
versions in our continuous integration system (:issue:`4504`)
.. _release-2.2.1:
Scrapy 2.2.1 (2020-07-17)
-------------------------
* The :command:`startproject` command no longer makes unintended changes to
the permissions of files in the destination folder, such as removing
execution permissions (:issue:`4662`, :issue:`4666`)
.. _release-2.2.0:
Scrapy 2.2.0 (2020-06-24)
@ -973,6 +1619,8 @@ Deprecation removals
* ``scrapy.xlib`` has been removed (:issue:`4015`)
.. _1.8-deprecations:
Deprecations
~~~~~~~~~~~~
@ -1329,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`):
* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
.. _1.7-deprecations:
Deprecations
~~~~~~~~~~~~
@ -1968,7 +2618,7 @@ Bug fixes
- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
Documentation
~~~~~~~~~~~~~
@ -3724,7 +4374,7 @@ API changes
- ``url`` and ``body`` attributes of Request objects are now read-only (#230)
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
- Changes to Scrapy Manager (now called "Crawler"):
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
@ -3871,13 +4521,14 @@ First release of Scrapy.
.. _resource: https://docs.python.org/2/library/resource.html
.. _robots.txt: https://www.robotstxt.org/
.. _scrapely: https://github.com/scrapy/scrapely
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
.. _six: https://six.readthedocs.io/
.. _tox: https://pypi.org/project/tox/
.. _Twisted: https://twistedmatrix.com/trac/
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _w3lib: https://github.com/scrapy/w3lib
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
.. _Zsh: https://www.zsh.org/
.. _zstandard: https://pypi.org/project/zstandard/

2
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx_rtd_theme>=0.4
sphinx-rtd-theme>=0.5.2

2
docs/topics/api.rst
View File

@ -4,8 +4,6 @@
Core API
========
.. versionadded:: 0.15
This section documents the Scrapy core API, and it's intended for developers of
extensions and middlewares.

5
docs/topics/architecture.rst
View File

@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the
Scheduler
---------
The Scheduler receives requests from the engine and enqueues them for feeding
them later (also to the engine) when the engine requests them.
The :ref:`scheduler <topics-scheduler>` receives requests from the engine and
enqueues them for feeding them later (also to the engine) when the engine
requests them.
.. _component-downloader:

25
docs/topics/asyncio.rst
View File

@ -1,16 +1,19 @@
.. _using-asyncio:
=======
asyncio
=======
.. versionadded:: 2.0
Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
reactor <install-asyncio>`, you may use :mod:`asyncio` and
Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related changes without a deprecation
period or warning.
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
recommended for production environments. Future Scrapy versions
may introduce related changes without a deprecation period or
warning.
.. _install-asyncio:
@ -26,3 +29,15 @@ reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
==========================
You can also use custom asyncio event loops with the asyncio reactor. Set the
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
use it instead of the default asyncio event loop.

2
docs/topics/autothrottle.rst
View File

@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies.
AUTOTHROTTLE_TARGET_CONCURRENCY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 1.1
Default: ``1.0``
Average number of requests Scrapy should be sending in parallel to remote

7
docs/topics/benchmarking.rst
View File

@ -4,8 +4,6 @@
Benchmarking
============
.. versionadded:: 0.17
Scrapy comes with a simple benchmarking suite that spawns a local HTTP server
and crawls it at the maximum possible speed. The goal of this benchmarking is
to get an idea of how Scrapy performs in your hardware, in order to have a
@ -83,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
results in slower crawl rates. How slower depends on how much your spider does
and how well it's written.
In the future, more cases will be added to the benchmarking suite to cover
other common scenarios.
Use scrapy-bench_ for more complex benchmarking.
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench

12
docs/topics/commands.rst
View File

@ -6,8 +6,6 @@
Command line tool
=================
.. versionadded:: 0.10
Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
here as the "Scrapy tool" to differentiate it from the sub-commands, which we
just call "commands" or "Scrapy commands".
@ -468,7 +466,7 @@ Supported options:
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
@ -491,6 +489,10 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
* ``--output`` or ``-o``: dump scraped items to a file
.. versionadded:: 2.3
.. skip: start
Usage example::
@ -562,8 +564,6 @@ and Platform info, which is useful for bug reports.
bench
-----
.. versionadded:: 0.17
* Syntax: ``scrapy bench``
* Requires project: *no*
@ -598,8 +598,6 @@ Example:
Register commands via setup.py entry points
-------------------------------------------
.. note:: This is an experimental feature, use with caution.
You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.

4
docs/topics/contracts.rst
View File

@ -4,8 +4,6 @@
Spiders Contracts
=================
.. versionadded:: 0.15
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@ -81,7 +79,7 @@ override three methods:
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function
:type method: collections.abc.Callable
:param args: list of arguments passed into the docstring (whitespace
separated)

15
docs/topics/coroutines.rst
View File

@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
The following are known caveats of the current implementation that we aim
to address in future versions of Scrapy:
- The callback output is not processed until the whole callback finishes.
.. note:: The callback output is not processed until the whole callback
finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
- Because `asynchronous generators were introduced in Python 3.6`_, you
can only use ``yield`` if you are using Python 3.6 or later.
If you need to output multiple items or requests and you are using
Python 3.5, return an iterable (e.g. a list) instead.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
Usage
=====

32
docs/topics/deploy.rst
View File

@ -14,7 +14,7 @@ spiders come in.
Popular choices for deploying Scrapy spiders are:
* :ref:`Scrapyd <deploy-scrapyd>` (open source)
* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
.. _deploy-scrapyd:
@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
.. _deploy-scrapy-cloud:
Deploying to Scrapy Cloud
=========================
Deploying to Zyte Scrapy Cloud
==============================
`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
the company behind Scrapy.
`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
behind Scrapy.
Scrapy Cloud removes the need to setup and monitor servers
and provides a nice UI to manage spiders and review scraped items,
logs and stats.
Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
nice UI to manage spiders and review scraped items, logs and stats.
To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
Please refer to the `Scrapy Cloud documentation`_ for more information.
To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
tool.
Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
Scrapy Cloud is compatible with Scrapyd and one can switch between
Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
them as needed - the configuration is read from the ``scrapy.cfg`` file
just like ``scrapyd-deploy``.
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
.. _shub: https://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: https://scrapinghub.com/
.. _shub: https://shub.readthedocs.io/en/latest/
.. _Zyte: https://zyte.com/
.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html

144
docs/topics/developer-tools.rst
View File

@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping
=================================================
Here is a general guide on how to use your browser's Developer Tools
to ease the scraping process. Today almost all browsers come with
to ease the scraping process. Today almost all browsers come with
built in `Developer Tools`_ and although we will use Firefox in this
guide, the concepts are applicable to any other browser.
guide, the concepts are applicable to any other browser.
In this guide we'll introduce the basic tools to use from a browser's
Developer Tools by scraping `quotes.toscrape.com`_.
@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things:
Inspecting a website
====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
any webpage. To demonstrate the Inspector, let's look at the
`quotes.toscrape.com`_-site.
On the site we have a total of ten quotes from various authors with specific
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
on this page, without any meta-information about authors, tags, etc.
Instead of viewing the whole source code for the page, we can simply right click
Instead of viewing the whole source code for the page, we can simply right click
on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
In it you should see something like this:
@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
go down a desired path in the source code starting from ``html``. So let's
see if we can refine our XPath a bit:
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
If we check the `Inspector` again we'll see that directly beneath our
expanded ``div`` tag we have nine identical ``div`` tags, each with the
same attributes as our first. If we expand any of them, we'll see the same
structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and
see each quote:
.. code-block:: html
@ -121,7 +121,7 @@ see each quote:
With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
@ -130,45 +130,45 @@ the `has-class-extension`_:
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
the number of the last ``div``, but this would have been unnecessarily
complex and by simply constructing an XPath with ``has-class("text")``
we were able to extract all quotes in one line.
we were able to extract all quotes in one line.
The `Inspector` has a lot of other helpful features, such as searching in the
The `Inspector` has a lot of other helpful features, such as searching in the
source code or directly scrolling to an element you selected. Let's demonstrate
a use case:
a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
follow the pagination. On a simple site such as this, there may not be
the need to find an element visually but the ``Scroll into View`` function
can be quite useful on complex sites.
can be quite useful on complex sites.
Note that the search bar can also be used to search for and test CSS
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
selectors. For example, you could search for ``span.text`` to find
all quote texts. Instead of a full text search, this searches for
exactly the ``span`` tag with the ``class="text"`` in the page.
.. _topics-network-tool:
The Network-tool
================
While scraping you may come across dynamic webpages where some parts
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
of the page are loaded dynamically through multiple requests. While
this can be quite tricky, the `Network`-tool in the Developer Tools
greatly facilitates this task. To demonstrate the Network-tool, let's
take a look at the page `quotes.toscrape.com/scroll`_.
take a look at the page `quotes.toscrape.com/scroll`_.
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
we'll check another quite useful command from the Scrapy shell:
.. skip: next
@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell:
(...)
>>> view(response)
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
A browser window should open with the webpage but with one
crucial difference: Instead of the quotes we just see a greenish
bar with the word ``Loading...``.
.. image:: _images/network_01.png
:width: 777
@ -189,21 +189,21 @@ bar with the word ``Loading...``.
:alt: Response from quotes.toscrape.com/scroll
The ``view(response)`` command let's us view the response our
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
shell or later our spider receives from the server. Here we see
that some basic template is loaded which includes the title,
the login-button and the footer, but the quotes are missing. This
tells us that the quotes are being loaded from a different request
than ``quotes.toscrape/scroll``.
than ``quotes.toscrape/scroll``.
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
If you click on the ``Network`` tab, you will probably only see
two entries. The first thing we do is enable persistent logs by
clicking on ``Persist Logs``. If this option is disabled, the
log is automatically cleared each time you navigate to a different
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
page. Enabling this option is a good default, since it gives us
control on when to clear the logs.
If we reload the page now, you'll see the log get populated with six
new requests.
new requests.
.. image:: _images/network_02.png
:width: 777
@ -212,31 +212,31 @@ new requests.
Here we see every request that has been made when reloading the page
and can inspect each request and its response. So let's find out
where our quotes are coming from:
where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
shell. Accordingly the ``type`` of the request in the log is ``html``.
The other requests have types like ``css`` or ``js``, but what
interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
on the request and open ``Open in new tab`` to get a better overview.
.. image:: _images/network_03.png
:width: 777
:height: 375
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
import scrapy
@ -255,17 +255,17 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
This spider starts at the first page of the quotes-API. With each
response, we parse the ``response.text`` and assign it to ``data``.
This lets us operate on the JSON-object like on a Python dictionary.
We iterate through the ``quotes`` and print out the ``quote["text"]``.
If the handy ``has_next`` element is ``true`` (try loading
If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
into our ``url``.
.. _requests-from-curl:
@ -289,14 +289,16 @@ request::
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments:
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.

65
docs/topics/downloader-middleware.rst
View File

@ -207,6 +207,11 @@ CookiesMiddleware
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`
@ -217,8 +222,6 @@ The following settings can be used to configure the cookie middleware:
Multiple cookie sessions per spider
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 0.15
There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
@ -475,8 +478,6 @@ DBM storage backend
.. class:: DbmCacheStorage
.. versionadded:: 0.13
A DBM_ storage backend is also available for the HTTP cache middleware.
By default, it uses the :mod:`dbm`, but you can change it with the
@ -549,15 +550,10 @@ settings:
HTTPCACHE_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.11
Default: ``False``
Whether the HTTP cache will be enabled.
.. versionchanged:: 0.11
Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
.. setting:: HTTPCACHE_EXPIRATION_SECS
HTTPCACHE_EXPIRATION_SECS
@ -570,9 +566,6 @@ Expiration time for cached requests, in seconds.
Cached requests older than this time will be re-downloaded. If zero, cached
requests will never expire.
.. versionchanged:: 0.11
Before 0.11, zero meant cached requests always expire.
.. setting:: HTTPCACHE_DIR
HTTPCACHE_DIR
@ -589,8 +582,6 @@ project data dir. For more info see: :ref:`topics-project-structure`.
HTTPCACHE_IGNORE_HTTP_CODES
^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``[]``
Don't cache response with these HTTP codes.
@ -609,8 +600,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade
HTTPCACHE_IGNORE_SCHEMES
^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.10
Default: ``['file']``
Don't cache responses with these URI schemes.
@ -629,8 +618,6 @@ The class which implements the cache storage backend.
HTTPCACHE_DBM_MODULE
^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
@ -641,8 +628,6 @@ The database module to use in the :ref:`DBM storage backend
HTTPCACHE_POLICY
^^^^^^^^^^^^^^^^
.. versionadded:: 0.18
Default: ``'scrapy.extensions.httpcache.DummyPolicy'``
The class which implements the cache policy.
@ -652,8 +637,6 @@ The class which implements the cache policy.
HTTPCACHE_GZIP
^^^^^^^^^^^^^^
.. versionadded:: 1.0
Default: ``False``
If enabled, will compress all cached data with gzip.
@ -664,8 +647,6 @@ This setting is specific to the Filesystem backend.
HTTPCACHE_ALWAYS_STORE
^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``False``
If enabled, will cache pages unconditionally.
@ -684,8 +665,6 @@ responses you feed to the cache middleware.
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. versionadded:: 1.1
Default: ``[]``
List of Cache-Control directives in responses to be ignored.
@ -710,11 +689,14 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses,
provided `brotlipy`_ is installed.
This middleware also supports decoding `brotli-compressed`_ as well as
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -735,8 +717,6 @@ HttpProxyMiddleware
.. module:: scrapy.downloadermiddlewares.httpproxy
:synopsis: Http Proxy Middleware
.. versionadded:: 0.8
.. reqmeta:: proxy
.. class:: HttpProxyMiddleware
@ -817,8 +797,6 @@ RedirectMiddleware settings
REDIRECT_ENABLED
^^^^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Redirect middleware will be enabled.
@ -860,8 +838,6 @@ MetaRefreshMiddleware settings
METAREFRESH_ENABLED
^^^^^^^^^^^^^^^^^^^
.. versionadded:: 0.17
Default: ``True``
Whether the Meta Refresh middleware will be enabled.
@ -916,6 +892,11 @@ settings (see the settings documentation for more info):
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
set to True, the request will be ignored by this middleware.
To retry requests from a spider callback, you can use the
:func:`get_retry_request` function:
.. autofunction:: get_retry_request
RetryMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~
@ -924,8 +905,6 @@ RetryMiddleware Settings
RETRY_ENABLED
^^^^^^^^^^^^^
.. versionadded:: 0.13
Default: ``True``
Whether the Retry middleware will be enabled.
@ -958,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
it is a common code used to indicate server overload. It is not included by
default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. _topics-dlmw-robots:
@ -1179,8 +1170,6 @@ AjaxCrawlMiddleware Settings
AJAXCRAWL_ENABLED
^^^^^^^^^^^^^^^^^
.. versionadded:: 0.21
Default: ``False``
Whether the AjaxCrawlMiddleware will be enabled. You may want to

14
docs/topics/email.rst
View File

@ -62,10 +62,10 @@ rest of the framework.
:type smtpport: int
:param smtptls: enforce using SMTP STARTTLS
:type smtptls: boolean
:type smtptls: bool
:param smtpssl: enforce using a secure SSL connection
:type smtpssl: boolean
:type smtpssl: bool
.. classmethod:: from_settings(settings)
@ -79,14 +79,14 @@ rest of the framework.
Send email to the given recipients.
:param to: the e-mail recipients
:type to: str or list of str
:param to: the e-mail recipients as a string or as a list of strings
:type to: str or list
:param subject: the subject of the e-mail
:type subject: str
:param cc: the e-mails to CC
:type cc: str or list of str
:param cc: the e-mails to CC as a string or as a list of strings
:type cc: str or list
:param body: the e-mail body
:type body: str
@ -96,7 +96,7 @@ rest of the framework.
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
attachment and ``file_object`` is a readable file object with the
contents of the attachment
:type attachs: iterable
:type attachs: collections.abc.Iterable
:param mimetype: the MIME type of the e-mail
:type mimetype: str

15
docs/topics/exceptions.rst
View File

@ -64,10 +64,10 @@ NotConfigured
This exception can be raised by some components to indicate that they will
remain disabled. Those components include:
* Extensions
* Item pipelines
* Downloader middlewares
* Spider middlewares
- Extensions
- Item pipelines
- Downloader middlewares
- Spider middlewares
The exception must be raised in the component's ``__init__`` method.
@ -85,8 +85,8 @@ StopDownload
.. exception:: StopDownload(fail=True)
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
indicate that no further bytes should be downloaded for a response.
Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signal handler to indicate that no further bytes should be downloaded for a response.
The ``fail`` boolean parameter controls which method will handle the resulting
response:
@ -110,5 +110,6 @@ attribute.
``StopDownload(False)`` or ``StopDownload(True)`` will raise
a :class:`TypeError`.
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
See the documentation for the :class:`~scrapy.signals.bytes_received` and
:class:`~scrapy.signals.headers_received` signals
and the :ref:`topics-stop-response-download` topic for additional information and examples.

34
docs/topics/exporters.rst
View File

@ -50,18 +50,19 @@ value of one of their fields::
self.year_to_exporter = {}
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
for exporter, xml_file in self.year_to_exporter.values():
exporter.finish_exporting()
xml_file.close()
def _exporter_for_item(self, item):
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
xml_file = open(f'{year}.xml', 'wb')
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
@ -98,7 +99,7 @@ Example::
import scrapy
def serialize_price(value):
return '$ %s' % str(value)
return f'$ {str(value)}'
class Product(scrapy.Item):
name = scrapy.Field()
@ -122,8 +123,8 @@ Example::
def serialize_field(self, field, name, value):
if field == 'price':
return '$ %s' % str(value)
return super(Product, self).serialize_field(field, name, value)
return f'$ {str(value)}'
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -166,8 +167,7 @@ BaseItemExporter
By default, this method looks for a serializer :ref:`declared in the item
field <topics-exporters-serializers>` and returns the result of applying
that serializer to the value. If no serializer is found, it returns the
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
value unchanged.
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
@ -217,10 +217,7 @@ BaseItemExporter
.. attribute:: encoding
The encoding that will be used to encode unicode values. This only
affects unicode values (which are always serialized to str using this
encoding). Other value types are passed unchanged to the specific
serialization library.
The output character encoding.
.. attribute:: indent
@ -296,7 +293,7 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs)
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
@ -309,12 +306,17 @@ CsvItemExporter
:param include_headers_line: If enabled, makes the exporter output a header
line with the field names taken from
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
:type include_headers_line: boolean
:type include_headers_line: bool
:param join_multivalued: The char (or chars) that will be used for joining
multi-valued fields, if found.
:type include_headers_line: str
:param errors: The optional string that specifies how encoding and decoding
errors are to be handled. For more information see
:class:`io.TextIOWrapper`.
:type errors: str
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
:func:`csv.writer` function, so you can use any :func:`csv.writer` function

19
docs/topics/extensions.rst
View File

@ -7,8 +7,7 @@ Extensions
The extensions framework provides a mechanism for inserting your own
custom functionality into Scrapy.
Extensions are just regular classes that are instantiated at Scrapy startup,
when extensions are initialized.
Extensions are just regular classes.
Extension settings
==================
@ -27,8 +26,8 @@ Loading & activating extensions
===============================
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
code must be performed in the class ``__init__`` method.
instance of the extension class per spider being run. All the extension
initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@ -257,6 +256,12 @@ settings:
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
When a certain closing condition is met, requests which are
currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS`
requests) are still processed.
.. setting:: CLOSESPIDER_TIMEOUT
CLOSESPIDER_TIMEOUT
@ -279,8 +284,6 @@ Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -288,8 +291,6 @@ If zero (or non set), spiders won't be closed by number of passed items.
CLOSESPIDER_PAGECOUNT
"""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of responses to crawl. If the spider
@ -302,8 +303,6 @@ number of crawled responses.
CLOSESPIDER_ERRORCOUNT
""""""""""""""""""""""
.. versionadded:: 0.11
Default: ``0``
An integer which specifies the maximum number of errors to receive before

354
docs/topics/feed-exports.rst
View File

@ -4,8 +4,6 @@
Feed exports
============
.. versionadded:: 0.10
One of the most frequently required features when implementing scrapers is
being able to store the scraped data properly and, quite often, that means
generating an "export file" with the scraped data (commonly called "export
@ -23,10 +21,10 @@ Serialization formats
For serializing the scraped data, the feed exports use the :ref:`Item exporters
<topics-exporters>`. These formats are supported out of the box:
* :ref:`topics-feed-format-json`
* :ref:`topics-feed-format-jsonlines`
* :ref:`topics-feed-format-csv`
* :ref:`topics-feed-format-xml`
- :ref:`topics-feed-format-json`
- :ref:`topics-feed-format-jsonlines`
- :ref:`topics-feed-format-csv`
- :ref:`topics-feed-format-xml`
But you can also extend the supported format through the
:setting:`FEED_EXPORTERS` setting.
@ -36,54 +34,58 @@ But you can also extend the supported format through the
JSON
----
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
- Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
- See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
.. _topics-feed-format-jsonlines:
JSON lines
----------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
CSV
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
.. _topics-feed-format-xml:
XML
---
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
- Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
Pickle
------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
- Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
Marshal
-------
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
.. _topics-feed-storage:
@ -97,10 +99,11 @@ storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-stdout`
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
@ -115,8 +118,8 @@ Storage URI parameters
The storage URI can also contain parameters that get replaced when the feed is
being created. These parameters are:
* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
* ``%(name)s`` - gets replaced by the spider name
- ``%(time)s`` - gets replaced by a timestamp when the feed is being created
- ``%(name)s`` - gets replaced by the spider name
Any other named parameter gets replaced by the spider attribute of the same
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
@ -124,13 +127,16 @@ attribute the moment the feed is being created.
Here are some examples to illustrate:
* Store in FTP using one directory per spider:
- Store in FTP using one directory per spider:
* ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
- ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
* Store in S3 using one directory per spider:
- Store in S3 using one directory per spider:
* ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
.. _topics-feed-storage-backends:
@ -145,9 +151,9 @@ Local filesystem
The feeds are stored in the local filesystem.
* URI scheme: ``file``
* Example URI: ``file:///tmp/export.csv``
* Required external libraries: none
- URI scheme: ``file``
- Example URI: ``file:///tmp/export.csv``
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
@ -160,15 +166,18 @@ FTP
The feeds are stored in a FTP server.
* URI scheme: ``ftp``
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
- URI scheme: ``ftp``
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- Required external libraries: none
FTP supports two different connection modes: `active or passive
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-s3:
S3
@ -176,23 +185,57 @@ S3
The feeds are stored on `Amazon S3`_.
* URI scheme: ``s3``
* Example URIs:
- URI scheme: ``s3``
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Example URIs:
* Required external libraries: `botocore`_
- ``s3://mybucket/path/to/export.csv``
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
You can also define a custom ACL for exported feeds using this setting:
* :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_STORAGE_S3_ACL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-gcs:
Google Cloud Storage (GCS)
--------------------------
.. versionadded:: 2.3
The feeds are stored on `Google Cloud Storage`_.
- URI scheme: ``gs``
- Example URIs:
- ``gs://mybucket/path/to/export.csv``
- Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
- :setting:`FEED_STORAGE_GCS_ACL`
- :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
.. _topics-feed-storage-stdout:
@ -201,9 +244,29 @@ Standard output
The feeds are written to the standard output of the Scrapy process.
* URI scheme: ``stdout``
* Example URI: ``stdout:``
* Required external libraries: none
- URI scheme: ``stdout``
- Example URI: ``stdout:``
- Required external libraries: none
.. _delayed-file-delivery:
Delayed file delivery
---------------------
As indicated above, some of the described storage backends use delayed file
delivery.
These storage backends do not upload items to the feed URI as those items are
scraped. Instead, Scrapy writes items into a temporary local file, and only
once all the file contents have been written (i.e. at the end of the crawl) is
that file uploaded to the feed URI.
If you want item delivery to start earlier when using one of these storage
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
in multiple files, with the specified maximum item count per file. That way, as
soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
Settings
@ -211,15 +274,16 @@ Settings
These are the settings used for configuring the feed exports:
* :setting:`FEEDS` (mandatory)
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
- :setting:`FEEDS` (mandatory)
- :setting:`FEED_EXPORT_ENCODING`
- :setting:`FEED_STORE_EMPTY`
- :setting:`FEED_EXPORT_FIELDS`
- :setting:`FEED_EXPORT_INDENT`
- :setting:`FEED_STORAGES`
- :setting:`FEED_STORAGE_FTP_ACTIVE`
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
@ -235,6 +299,7 @@ Default: ``{}``
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
This setting is required for enabling the feed export feature.
See :ref:`topics-feed-storage-backends` for supported URI schemes.
@ -248,6 +313,9 @@ For instance::
'store_empty': False,
'fields': None,
'indent': 4,
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
@ -261,16 +329,54 @@ For instance::
},
}
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition.
.. _feed-options:
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition:
- ``format``: the :ref:`serialization format <topics-feed-format>`.
This setting is mandatory, there is no fallback value.
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
.. versionadded:: 2.4.0
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
The default value depends on the :ref:`storage backend
<topics-feed-storage-backends>`:
- :ref:`topics-feed-storage-fs`: ``False``
- :ref:`topics-feed-storage-ftp`: ``True``
.. note:: Some FTP servers may not support appending to files (the
``APPE`` FTP command).
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
.. versionadded:: 2.4.0
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
* ``format``: the serialization format to be used for the feed.
See :ref:`topics-feed-format` for possible values.
Mandatory, no fallback setting
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
.. setting:: FEED_EXPORT_ENCODING
@ -425,7 +531,117 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
----------------------------
.. versionadded:: 2.3.0
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
placeholders in the feed URI to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
alter the number format. For example, to make the batch ID a 5-digit
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
FEED_EXPORT_BATCH_ITEM_COUNT = 100
And your :command:`crawl` command line is::
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
The command line above can generate a directory tree like::
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fewer.
.. setting:: FEED_URI_PARAMS
FEED_URI_PARAMS
---------------
Default: ``None``
A string with the import path of a function to set the parameters to apply with
:ref:`printf-style string formatting <python:old-string-formatting>` to the
feed URI.
The function signature should be as follows:
.. function:: uri_params(params, spider)
Return a :class:`dict` of key-value pairs to apply to the feed URI using
:ref:`printf-style string formatting <python:old-string-formatting>`.
:param params: default key-value pairs
Specifically:
- ``batch_id``: ID of the file batch. See
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
.. versionadded:: 2.3.0
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
.. versionadded:: 2.3.0
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.spiders.Spider
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

6
docs/topics/item-pipeline.rst
View File

@ -96,7 +96,7 @@ contain a price::
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
@ -211,7 +211,7 @@ item.
# Save screenshot to file, filename will be hash of url.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
@ -240,7 +240,7 @@ returns multiples items with the same id::
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item)
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
return item

2
docs/topics/jobs.rst
View File

@ -65,7 +65,7 @@ Cookies expiration
------------------
Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
scheduled may no longer work. This won't be an issue if your spider doesn't rely
on cookies.

4
docs/topics/leaks.rst
View File

@ -102,7 +102,7 @@ A real example
Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
:param ignore: if given, all objects from the specified class (or tuple of
classes) will be ignored.
:type ignore: class or classes tuple
:type ignore: type or tuple
.. function:: get_oldest(class_name)

35
docs/topics/link-extractors.rst
View File

@ -10,12 +10,19 @@ The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
list of matching :class:`~scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
def parse(self, response):
for link in self.link_extractor.extract_links(response):
yield Request(link.url, callback=self.parse)
.. _topics-link-extractors-ref:
@ -46,13 +53,13 @@ LxmlLinkExtractor
:param allow: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be extracted. If not
given (or empty), it will match all links.
:type allow: a regular expression (or list of)
:type allow: str or list
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
:type deny: str or list
:param allow_domains: a single value or a list of string containing
domains which will be considered for extracting the links
@ -88,7 +95,7 @@ LxmlLinkExtractor
that the link's text must match in order to be extracted. If not
given (or empty), it will match all links. If a list of regular expressions is
given, the link will be extracted if it matches at least one.
:type restrict_text: a regular expression (or list of)
:type restrict_text: str or list
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
@ -106,11 +113,11 @@ LxmlLinkExtractor
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:type canonicalize: bool
:param unique: whether duplicate filtering should be applied to extracted
links.
:type unique: boolean
:type unique: bool
:param process_value: a function which receives each value extracted from
the tag and attributes scanned and can modify the value and return a
@ -132,7 +139,7 @@ LxmlLinkExtractor
if m:
return m.group(1)
:type process_value: callable
:type process_value: collections.abc.Callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
@ -141,8 +148,16 @@ LxmlLinkExtractor
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
:type strip: bool
.. automethod:: extract_links
Link
----
.. module:: scrapy.link
:synopsis: Link from link extractors
.. autoclass:: Link
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

452
docs/topics/loaders.rst
View File

@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
for extending and overriding different field parsing rules, either by spider,
or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
.. note:: Item Loaders are an extension of the itemloaders_ library that make it
easier to work with Scrapy by adding support for
:ref:`responses <topics-request-response>`.
Using Item Loaders to populate items
====================================
@ -88,29 +92,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
Given the way that item loaders store data internally, one approach
to overcome this is to define items using the :func:`~dataclasses.field`
function, with ``list`` as the ``default_factory`` argument::
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: str = field(default_factory=list)
price: float = field(default_factory=list)
stock: int = field(default_factory=list)
Note that in order to keep the example simple, the types do not match
completely. A more accurate but verbose definition would be::
from dataclasses import dataclass, field
from typing import List, Union
@dataclass
class InventoryItem:
name: Union[str, List[str]] = field(default_factory=list)
price: Union[float, List[float]] = field(default_factory=list)
stock: Union[int, List[int]] = field(default_factory=list)
name: Optional[str] = field(default=None)
price: Optional[float] = field(default=None)
stock: Optional[int] = field(default=None)
.. _topics-loaders-processors:
@ -185,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
processors are collected internally (in lists) and then passed to output
processors to populate the fields.
Last, but not least, Scrapy comes with some :ref:`commonly used processors
<topics-loaders-available-processors>` built-in for convenience.
Last, but not least, itemloaders_ comes with some :ref:`commonly used
processors <itemloaders:built-in-processors>` built-in for convenience.
Declaring Item Loaders
@ -194,17 +186,17 @@ Declaring Item Loaders
Item Loaders are declared using a class definition syntax. Here is an example::
from itemloaders.processors import TakeFirst, MapCompose, Join
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose, Join
class ProductLoader(ItemLoader):
default_output_processor = TakeFirst()
name_in = MapCompose(unicode.title)
name_in = MapCompose(str.title)
name_out = Join()
price_in = MapCompose(unicode.strip)
price_in = MapCompose(str.strip)
# ...
@ -226,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
metadata. Here is an example::
import scrapy
from scrapy.loader.processors import Join, MapCompose, TakeFirst
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_price(value):
@ -245,10 +237,10 @@ metadata. Here is an example::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:
@ -307,250 +299,9 @@ There are several ways to modify Item Loader context values:
ItemLoader objects
==================
.. class:: ItemLoader([item, selector, response], **kwargs)
Return a new Item Loader for populating the given :ref:`item object
<topics-items>`. If no item object is given, one is instantiated
automatically using the class in :attr:`default_item_class`.
When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
data from web pages using :ref:`selectors <topics-selectors>`.
:param item: The item instance to populate using subsequent calls to
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
or :meth:`~ItemLoader.add_value`.
:type item: :ref:`item object <topics-items>`
:param selector: The selector to extract data from, when using the
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
(resp. :meth:`replace_css`) method.
:type selector: :class:`~scrapy.selector.Selector` object
:param response: The response used to construct the selector using the
:attr:`default_selector_class`, unless the selector argument is given,
in which case this argument is ignored.
:type response: :class:`~scrapy.http.Response` object
The item, selector, response and the remaining keyword arguments are
assigned to the Loader context (accessible through the :attr:`context` attribute).
:class:`ItemLoader` instances have the following methods:
.. method:: get_value(value, *processors, **kwargs)
Process the given ``value`` by the given ``processors`` and keyword
arguments.
Available keyword arguments:
:param re: a regular expression to use for extracting data from the
given value using :meth:`~scrapy.utils.misc.extract_regex` method,
applied before processors
:type re: str or compiled regex
Examples:
>>> from scrapy.loader.processors import TakeFirst
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, *processors, **kwargs)
Process and then add the given ``value`` for the given field.
The value is first passed through :meth:`get_value` by giving the
``processors`` and ``kwargs``, and then passed through the
:ref:`field input processor <topics-loaders-processors>` and its result
appended to the data collected for that field. If the field already
contains collected data, the new data is added.
The given ``field_name`` can be ``None``, in which case values for
multiple fields may be added. And the processed value should be a dict
with field_name mapped to values.
Examples::
loader.add_value('name', u'Color TV')
loader.add_value('colours', [u'white', u'blue'])
loader.add_value('length', u'100')
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
.. method:: replace_value(field_name, value, *processors, **kwargs)
Similar to :meth:`add_value` but replaces the collected data with the
new value instead of adding it.
.. method:: get_xpath(xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
selector associated with this :class:`ItemLoader`.
:param xpath: the XPath to extract data from
:type xpath: str
:param re: a regular expression to use for extracting data from the
selected XPath region
:type re: str or compiled regex
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.get_xpath('//p[@class="product-name"]')
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
selector associated with this :class:`ItemLoader`.
See :meth:`get_xpath` for ``kwargs``.
:param xpath: the XPath to extract data from
:type xpath: str
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.add_xpath('name', '//p[@class="product-name"]')
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`add_xpath` but replaces collected data instead of
adding it.
.. method:: get_css(css, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
from the selector associated with this :class:`ItemLoader`.
:param css: the CSS selector to extract data from
:type css: str
:param re: a regular expression to use for extracting data from the
selected CSS region
:type re: str or compiled regex
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.get_css('p.product-name')
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
.. method:: add_css(field_name, css, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
from the selector associated with this :class:`ItemLoader`.
See :meth:`get_css` for ``kwargs``.
:param css: the CSS selector to extract data from
:type css: str
Examples::
# HTML snippet: <p class="product-name">Color TV</p>
loader.add_css('name', 'p.product-name')
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_css('price', 'p#price', re='the price is (.*)')
.. method:: replace_css(field_name, css, *processors, **kwargs)
Similar to :meth:`add_css` but replaces collected data instead of
adding it.
.. method:: load_item()
Populate the item with the data collected so far, and return it. The
data collected is first passed through the :ref:`output processors
<topics-loaders-processors>` to get the final value to assign to each
item field.
.. method:: nested_xpath(xpath)
Create a nested loader with an xpath selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: nested_css(css)
Create a nested loader with a css selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: get_collected_values(field_name)
Return the collected values for the given field.
.. method:: get_output_value(field_name)
Return the collected values parsed using the output processor, for the
given field. This method doesn't populate or modify the item at all.
.. method:: get_input_processor(field_name)
Return the input processor for the given field.
.. method:: get_output_processor(field_name)
Return the output processor for the given field.
:class:`ItemLoader` instances have the following attributes:
.. attribute:: item
The :ref:`item object <topics-items>` being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
The currently active :ref:`Context <topics-loaders-context>` of this
Item Loader.
.. attribute:: default_item_class
An :ref:`item object <topics-items>` class or factory, used to
instantiate items when not given in the ``__init__`` method.
.. attribute:: default_input_processor
The default input processor to use for those fields which don't specify
one.
.. attribute:: default_output_processor
The default output processor to use for those fields which don't specify
one.
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
It's either the selector given in the ``__init__`` method or one created from
the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
.. autoclass:: scrapy.loader.ItemLoader
:members:
:inherited-members:
.. _topics-loaders-nested:
@ -621,7 +372,7 @@ those dashes in the final product names.
Here's how you can remove those dashes by reusing and extending the default
Product Item Loader (``ProductLoader``)::
from scrapy.loader.processors import MapCompose
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
def strip_dashes(x):
@ -634,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
multiple source formats, for example XML and HTML. In the XML version you may
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
from scrapy.loader.processors import MapCompose
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
from myproject.utils.xml import remove_cdata
@ -654,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
organization of your Loaders collection - that's up to you and your project's
needs.
.. _topics-loaders-available-processors:
Available built-in processors
=============================
.. module:: scrapy.loader.processors
:synopsis: A collection of processors to use with Item Loaders
Even though you can use any callable function as input and output processors,
Scrapy provides some commonly used processors, which are described below. Some
of them, like the :class:`MapCompose` (which is typically used as input
processor) compose the output of several functions executed in order, to
produce the final parsed value.
Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example:
>>> from scrapy.loader.processors import Identity
>>> proc = Identity()
>>> proc(['one', 'two', 'three'])
['one', 'two', 'three']
.. class:: TakeFirst
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example:
>>> from scrapy.loader.processors import TakeFirst
>>> proc = TakeFirst()
>>> proc(['', 'one', 'two', 'three'])
'one'
.. class:: Join(separator=u' ')
Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
function: ``u' '.join``
Examples:
>>> from scrapy.loader.processors import Join
>>> proc = Join()
>>> proc(['one', 'two', 'three'])
'one two three'
>>> proc = Join('<br>')
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions. This means that each input value of this processor is passed to
the first function, and the result of that function is passed to the second
function, and so on, until the last function returns the output value of
this processor.
By default, stop process on ``None`` value. This behaviour can be changed by
passing keyword argument ``stop_on_none=False``.
Example:
>>> from scrapy.loader.processors import Compose
>>> proc = Compose(lambda v: v[0], str.upper)
>>> proc(['hello', 'world'])
'HELLO'
Each function can optionally receive a ``loader_context`` parameter. For
those which do, this processor will pass the currently active :ref:`Loader
context <topics-loaders-context>` through that parameter.
The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
attribute.
.. class:: MapCompose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions, similar to the :class:`Compose` processor. The difference with
this processor is the way internal results are passed among functions,
which is as follows:
The input value of this processor is *iterated* and the first function is
applied to each element. The results of these function calls (one for each element)
are concatenated to construct a new iterable, which is then used to apply the
second function, and so on, until the last function is applied to each
value of the list of values collected so far. The output values of the last
function are concatenated together to produce the output of this processor.
Each particular function can return a value or a list of values, which is
flattened with the list of values returned by the same function applied to
the other input values. The functions can also return ``None`` in which
case the output of that function is ignored for further processing over the
chain.
This processor provides a convenient way to compose functions that only
work with single values (instead of iterables). For this reason the
:class:`MapCompose` processor is typically used as input processor, since
data is often extracted using the
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
<topics-selectors>`, which returns a list of unicode strings.
The example below should clarify how it works:
>>> def filter_world(x):
... return None if x == 'world' else x
...
>>> from scrapy.loader.processors import MapCompose
>>> proc = MapCompose(filter_world, str.upper)
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
Example:
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
>>> proc({'foo': 'bar'})
'bar'
>>> proc({'foo': {'bar': 'baz'}})
{'bar': 'baz'}
Working with Json:
>>> import json
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
>>> proc_single_json_str('{"foo": "bar"}')
'bar'
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
['bar']
.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html

45
docs/topics/logging.rst
View File

@ -101,7 +101,7 @@ instance, which can be accessed and used like this::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
@ -117,7 +117,7 @@ Python logger you want. For example::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapinghub.com']
start_urls = ['https://scrapy.org']
def parse(self, response):
logger.info('Parse function called on %s', response.url)
@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method::
If you run this spider again then INFO messages from
``scrapy.spidermiddlewares.httperror`` logger will be gone.
You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================

58
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -56,6 +56,8 @@ this:
error will be logged and the file won't be present in the ``files`` field.
.. _images-pipeline:
Using the Images Pipeline
=========================
@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
JPEG/RGB format, so you need to install this library in order to use it.
`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
PIL.
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
.. _Python Imaging Library: http://www.pythonware.com/products/pil/
.. _topics-media-pipeline-enabling:
@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
the passive connection mode by default. To use the active connection mode instead,
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
.. _media-pipelines-s3:
Amazon S3 storage
-----------------
.. setting:: FILES_STORE_S3_ACL
.. setting:: IMAGES_STORE_S3_ACL
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
bucket. Scrapy will automatically upload the files to the bucket.
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
@ -187,8 +188,9 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
@ -207,7 +210,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
Google Cloud Storage
---------------------
.. setting:: GCS_PROJECT_ID
.. setting:: FILES_STORE_GCS_ACL
.. setting:: IMAGES_STORE_GCS_ACL
@ -413,15 +415,16 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -437,12 +440,18 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@ -545,15 +554,16 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(self, request, response=None, info=None)
.. method:: file_path(self, request, response=None, info=None, *, item=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>` and
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
:class:`request <scrapy.Request>`,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.item.Item>`
You can override this method to customize the download path of each file.
@ -569,12 +579,18 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
Similarly, you can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`file_path` method returns
``full/<request URL hash>.<extension>``.
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,

18
docs/topics/practices.rst
View File

@ -63,7 +63,7 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapinghub.com')
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
@ -119,6 +119,7 @@ Here is an example that runs multiple spiders simultaneously:
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously:
# Your second spider definition
...
process = CrawlerProcess()
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
@ -141,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -151,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
runner = CrawlerRunner(settings)
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
@ -166,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
@ -176,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
...
configure_logging()
runner = CrawlerRunner()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
@ -244,7 +250,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
`Crawlera`_
`Zyte Smart Proxy Manager`_
If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
@ -254,5 +260,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

178
docs/topics/request-response.rst
View File

@ -26,14 +26,10 @@ Request objects
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: string
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
@ -42,25 +38,31 @@ Request objects
:meth:`~scrapy.spiders.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
:type callback: callable
:type callback: collections.abc.Callable
:param method: the HTTP method of this request. Defaults to ``'GET'``.
:type method: string
:type method: str
:param meta: the initial values for the :attr:`Request.meta` attribute. If
given, the dict passed in this parameter will be shallow copied.
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
:type body: str or unicode
:param body: the request body. If a string is passed, then it's encoded as
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty bytes object is stored. Regardless of the
type of this argument, the final value stored will be a bytes object
(never a string or ``None``).
:type body: bytes or str
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
``None`` is passed as value, the HTTP header will not be sent at all.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
:type headers: dict
:param cookies: the request cookies. These can be sent in two forms.
@ -102,12 +104,18 @@ Request objects
)
For more info see :ref:`cookies-mw`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
:type cookies: dict or list
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to ``str`` (if given as ``unicode``).
:type encoding: string
body to bytes (if given as a string).
:type encoding: str
:param priority: the priority of this request (defaults to ``0``).
The priority is used by the scheduler to define the order used to process
@ -119,7 +127,7 @@ Request objects
the scheduler. This is used when you want to perform an identical
request multiple times, to ignore the duplicates filter. Use it with
care, or you will get into crawling loops. Default to ``False``.
:type dont_filter: boolean
:type dont_filter: bool
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
@ -131,7 +139,7 @@ Request objects
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: callable
:type errback: collections.abc.Callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
@ -159,7 +167,7 @@ Request objects
.. attribute:: Request.body
A str that contains the request body.
The request body as bytes.
This attribute is read-only. To change the body of a Request use
:meth:`replace`.
@ -193,6 +201,8 @@ Request objects
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. autoattribute:: Request.attributes
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@ -208,6 +218,15 @@ Request objects
.. automethod:: from_curl
.. automethod:: to_dict
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.utils.request.request_from_dict
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -351,26 +370,26 @@ are some special keys recognized by Scrapy and its built-in extensions.
Those are:
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`bindaddress`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`max_retry_times`
* :reqmeta:`proxy`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`download_timeout`
* :reqmeta:`download_maxsize`
* :reqmeta:`download_latency`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`proxy`
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* :reqmeta:`referrer_policy`
* :reqmeta:`max_retry_times`
.. reqmeta:: bindaddress
@ -420,9 +439,9 @@ The meta key is used set retry times per request. When initialized, the
Stopping the download of a Response
===================================
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
:class:`~scrapy.signals.bytes_received` signal handler will stop the
download of a given response. See the following example::
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
import scrapy
@ -485,7 +504,7 @@ fields with form data from :class:`Response` objects.
:param formdata: is a dictionary (or iterable of (key, value) tuples)
containing HTML Form data which will be url-encoded and assigned to the
body of the request.
:type formdata: dict or iterable of tuples
:type formdata: dict or collections.abc.Iterable
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
@ -517,20 +536,20 @@ fields with form data from :class:`Response` objects.
:type response: :class:`Response` object
:param formname: if given, the form with name attribute set to this value will be used.
:type formname: string
:type formname: str
:param formid: if given, the form with id attribute set to this value will be used.
:type formid: string
:type formid: str
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
:type formxpath: str
:param formcss: if given, the first form that matches the css selector will be used.
:type formcss: string
:type formcss: str
:param formnumber: the number of form to use, when the response contains
multiple forms. The first one (and also the default) is ``0``.
:type formnumber: integer
:type formnumber: int
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
@ -548,23 +567,11 @@ fields with form data from :class:`Response` objects.
:param dont_click: If True, the form data will be submitted without
clicking in any element.
:type dont_click: boolean
:type dont_click: bool
The other parameters of this class method are passed directly to the
:class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
.. versionadded:: 0.17
The ``formxpath`` parameter.
.. versionadded:: 1.1.0
The ``formcss`` parameter.
.. versionadded:: 1.1.0
The ``formid`` parameter.
Request usage examples
----------------------
@ -636,12 +643,14 @@ dealing with JSON requests.
if :attr:`Request.body` argument is provided this parameter will be ignored.
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
:type data: object
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
:type dumps_kwargs: dict
.. autoattribute:: JsonRequest.attributes
JsonRequest usage example
-------------------------
@ -663,16 +672,16 @@ Response objects
downloaded (by the Downloader) and fed to the Spiders for processing.
:param url: the URL of this response
:type url: string
:type url: str
:param status: the HTTP status of the response. Defaults to ``200``.
:type status: integer
:type status: int
:param headers: the headers of this response. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. To access the decoded text as str you can use
:param body: the response body. To access the decoded text as a string, use
``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
@ -693,9 +702,19 @@ Response objects
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
:param protocol: The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1", "h2"
:type protocol: :class:`str`
.. versionadded:: 2.0.0
The ``certificate`` parameter.
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. versionadded:: 2.5.0
The ``protocol`` parameter.
.. attribute:: Response.url
A string containing the URL of the response.
@ -720,10 +739,10 @@ Response objects
.. attribute:: Response.body
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the unicode version use
:attr:`TextResponse.text` (only available in :class:`TextResponse`
and subclasses).
The response body as bytes.
If you want the body as a string, use :attr:`TextResponse.text` (only
available in :class:`TextResponse` and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@ -780,6 +799,8 @@ Response objects
.. attribute:: Response.certificate
.. versionadded:: 2.0.0
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
@ -795,6 +816,17 @@ Response objects
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. attribute:: Response.protocol
.. versionadded:: 2.5.0
The protocol that was used to download the response.
For instance: "HTTP/1.0", "HTTP/1.1"
This attribute is currently only populated by the HTTP download
handlers, i.e. for ``http(s)`` responses. For other handlers,
:attr:`protocol` is always ``None``.
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -842,18 +874,18 @@ TextResponse objects
is the same as for the :class:`Response` class and is not documented here.
:param encoding: is a string which contains the encoding to use for this
response. If you create a :class:`TextResponse` object with a unicode
body, it will be encoded using this encoding (remember the body attribute
is always a string). If ``encoding`` is ``None`` (default value), the
encoding will be looked up in the response headers and body instead.
:type encoding: string
response. If you create a :class:`TextResponse` object with a string as
body, it will be converted to bytes encoded using this encoding. If
*encoding* is ``None`` (default), the encoding will be looked up in the
response headers and body instead.
:type encoding: str
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:
.. attribute:: TextResponse.text
Response body, as unicode.
Response body, as a string.
The same as ``response.body.decode(response.encoding)``, but the
result is cached after the first call, so you can access
@ -861,9 +893,11 @@ TextResponse objects
.. note::
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically ``ascii``) instead of the response encoding.
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. attribute:: TextResponse.encoding

34
docs/topics/scheduler.rst Normal file
View File

@ -0,0 +1,34 @@
.. _topics-scheduler:
=========
Scheduler
=========
.. module:: scrapy.core.scheduler
The scheduler component receives requests from the :ref:`engine <component-engine>`
and stores them into persistent and/or non-persistent data structures.
It also gets those requests and feeds them back to the engine when it
asks for a next request to be downloaded.
Overriding the default scheduler
================================
You can use your own custom scheduler class by supplying its full
Python path in the :setting:`SCHEDULER` setting.
Minimal scheduler interface
===========================
.. autoclass:: BaseScheduler
:members:
Default Scrapy scheduler
========================
.. autoclass:: Scheduler
:members:
:special-members: __len__

36
docs/topics/selectors.rst
View File

@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
- `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
HTML code and also deals with bad markup reasonably well, but it has one
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
library.)
- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
standard library.)
Scrapy comes with its own mechanism for extracting data. They're called
selectors because they "select" certain parts of the HTML document specified
@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
markup as an unicode string (in ``text`` argument).
markup as a string (in ``text`` argument).
Usually there is no need to construct Scrapy selectors manually:
``response`` object is available in Spider callbacks, so in most cases
it is more convenient to use ``response.css()`` and ``response.xpath()``
@ -327,8 +328,9 @@ too. Here's an example:
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
... print('Link number %d points to url %r and image %r' % args)
... href_xpath = link.xpath('@href').get()
... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
@ -383,7 +385,7 @@ Using selectors with regular expressions
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
data using regular expressions. However, unlike using ``.xpath()`` or
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
``.css()`` methods, ``.re()`` returns a list of strings. So you
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
@ -462,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
you may want to take a look first at this `XPath tutorial`_.
.. note::
Some of the tips are based on `this post from ScrapingHub's blog`_.
Some of the tips are based on `this post from Zyte's blog`_.
.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
.. _topics-selectors-relative-xpaths:
@ -734,7 +736,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = u"""
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
@ -765,7 +767,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
>>> doc = u"""
>>> doc = """
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@ -821,7 +823,7 @@ with groups of itemscopes and corresponding itemprops::
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
... print(" properties: %s" % (props.getall()))
... print(f" properties: {props.getall()}")
... print("")
current scope: ['http://schema.org/Product']
@ -989,7 +991,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel.xpath("//h1")
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
returning a list of strings::
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag

152
docs/topics/settings.rst
View File

@ -98,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Import paths and classes
========================
.. versionadded:: 2.4.0
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
- As a string containing the import path of that object
- As the object itself
For example::
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
}
.. note:: Passing non-callable objects is not supported.
How to access settings
======================
@ -110,7 +136,7 @@ In a spider, the settings are available through ``self.settings``::
start_urls = ['http://example.com']
def parse(self, response):
print("Existing settings: %s" % self.settings.attributes.keys())
print(f"Existing settings: {self.settings.attributes.keys()}")
.. note::
The ``settings`` attribute is set in the base Spider class after the spider
@ -216,6 +242,32 @@ Default: ``None``
The name of the region associated with the AWS client.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
------------------
Default: ``None``
Import path of a given ``asyncio`` event loop class.
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
asyncio event loop to be used with it. Set the setting to the import path of the
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
event loop will be used.
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
class to be used.
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
.. caution:: Please be aware that, when using a non-default event loop
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
:func:`asyncio.set_event_loop`, which will set the specified event loop
as the current loop for the current OS thread.
.. setting:: BOT_NAME
BOT_NAME
@ -306,6 +358,11 @@ Default::
The default headers used for Scrapy HTTP Requests. They're populated in the
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
:ref:`cookies-mw`. If you need to set cookies for a request, use the
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
current limitation that is being worked on.
.. setting:: DEPTH_LIMIT
DEPTH_LIMIT
@ -469,7 +526,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@ -600,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE
Default::
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@ -620,6 +678,45 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
.. _http2:
The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
}
.. warning::
HTTP/2 support in Scrapy is experimental, and not yet recommended for
production environments. Future Scrapy versions may introduce related
changes without a deprecation period or warning.
.. note::
Known limitations of the current HTTP/2 implementation of Scrapy include:
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
- No setting to specify a maximum `frame size`_ larger than the default
value, 16384. Connections to servers that send a larger frame will
fail.
- No support for `server pushes`_, which are ignored.
- No support for the :signal:`bytes_received` and
:signal:`headers_received` signals.
.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -697,6 +794,15 @@ Optionally, this can be set per-request basis by using the
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
.. warning::
This setting is ignored by the
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
error, the corresponding HTTP/2 connection may be corrupted, affecting other
requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
failure is always raised for every request that was using that connection.
.. setting:: DUPEFILTER_CLASS
DUPEFILTER_CLASS
@ -786,6 +892,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
:ref:`Amazon S3 <topics-feed-storage-s3>`.
.. setting:: FEED_STORAGE_GCS_ACL
FEED_STORAGE_GCS_ACL
--------------------
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
.. setting:: FTP_PASSIVE_MODE
FTP_PASSIVE_MODE
@ -825,6 +939,15 @@ Default: ``"anonymous"``
The username to use for FTP connections when there is no ``"ftp_user"``
in ``Request`` meta.
.. setting:: GCS_PROJECT_ID
GCS_PROJECT_ID
-----------------
Default: ``None``
The Project ID that will be used when storing data on `Google Cloud Storage`_.
.. setting:: ITEM_PIPELINES
ITEM_PIPELINES
@ -1013,8 +1136,6 @@ See :ref:`topics-extensions-ref-memusage`.
MEMUSAGE_CHECK_INTERVAL_SECONDS
-------------------------------
.. versionadded:: 1.1
Default: ``60.0``
Scope: ``scrapy.extensions.memusage``
@ -1116,20 +1237,6 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
Default: ``-1``
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
Adjust retry request priority relative to original request:
- a positive priority adjust means higher priority.
- **a negative priority adjust (default) means lower priority.**
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY
@ -1177,7 +1284,8 @@ SCHEDULER
Default: ``'scrapy.core.scheduler.Scheduler'``
The scheduler to use for crawling.
The scheduler class to be used for crawling.
See the :ref:`topics-scheduler` topic for details.
.. setting:: SCHEDULER_DEBUG
@ -1295,8 +1403,6 @@ The class that will be used for loading spiders, which must implement the
SPIDER_LOADER_WARN_ONLY
-----------------------
.. versionadded:: 1.3.3
Default: ``False``
By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
@ -1517,7 +1623,7 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@ -1540,7 +1646,7 @@ case to see how to enable and use them.
.. settingslist::
.. _Amazon web services: https://aws.amazon.com/
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
.. _Google Cloud Storage: https://cloud.google.com/storage/

45
docs/topics/shell.rst
View File

@ -95,20 +95,21 @@ convenience.
Available Shortcuts
-------------------
* ``shelp()`` - print a help with the list of available objects and shortcuts
- ``shelp()`` - print a help with the list of available objects and
shortcuts
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
URL and update all related objects accordingly. You can optionaly ask for
HTTP 3xx redirections to not be followed by passing ``redirect=False``
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
and update all related objects accordingly. You can optionaly ask for HTTP
3xx redirections to not be followed by passing ``redirect=False``
* ``fetch(request)`` - fetch a new response from the given request and
update all related objects accordingly.
- ``fetch(request)`` - fetch a new response from the given request and update
all related objects accordingly.
* ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
- ``view(response)`` - open the given response in your local web browser, for
inspection. This will add a `\<base\> tag`_ to the response body in order
for external links (such as images and style sheets) to display properly.
Note, however, that this will create a temporary file in your computer,
which won't be removed automatically.
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
@ -122,21 +123,21 @@ content).
Those objects are:
* ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
* ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for
the current URL
- ``spider`` - the Spider which is known to handle the URL, or a
:class:`~scrapy.spiders.Spider` object if there is no spider found for the
current URL
* ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using :meth:`~scrapy.http.Request.replace`
or fetch a new request (without leaving the shell) using the ``fetch``
shortcut.
- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
page. You can modify this request using
:meth:`~scrapy.http.Request.replace` or fetch a new request (without
leaving the shell) using the ``fetch`` shortcut.
* ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
- ``response`` - a :class:`~scrapy.http.Response` object containing the last
fetched page
* ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
- ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
Example of shell session
========================

37
docs/topics/signals.rst
View File

@ -384,6 +384,11 @@ bytes_received
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
@ -395,11 +400,36 @@ bytes_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: Handlers of this signal can stop the download of a response while it
headers_received
~~~~~~~~~~~~~~~~
.. versionadded:: 2.5
.. signal:: headers_received
.. function:: headers_received(headers, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
available for a given request, before downloading any additional content.
Handlers for this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
This signal does not support returning deferreds from its handlers.
:param headers: the headers received by the download handler
:type headers: :class:`scrapy.http.headers.Headers` object
:param body_length: expected size of the response body, in bytes
:type body_length: `int`
:param request: the request that generated the download
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
Response signals
----------------
@ -423,6 +453,11 @@ response_received
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: The ``request`` argument might not contain the original request that
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
the :class:`~scrapy.http.Response` object and sets a specific ``request``
attribute.
response_downloaded
~~~~~~~~~~~~~~~~~~~

9
docs/topics/spider-middleware.rst
View File

@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
.. method:: process_start_requests(start_requests, spider)
.. versionadded:: 0.15
This method is called with the start requests of the spider, and works
similarly to the :meth:`process_spider_output` method, except that it
doesn't have a response associated and must return only requests (not
@ -255,7 +253,8 @@ this::
The ``handle_httpstatus_list`` key of :attr:`Request.meta
<scrapy.http.Request.meta>` can also be used to specify which response codes to
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
to ``True`` if you want to allow any response code for a request.
to ``True`` if you want to allow any response code for a request, and ``False`` to
disable the effects of the ``handle_httpstatus_all`` key.
Keep in mind, however, that it's usually a bad idea to handle non-200
responses, unless you really know what you're doing.
@ -341,8 +340,6 @@ RefererMiddleware settings
REFERER_ENABLED
^^^^^^^^^^^^^^^
.. versionadded:: 0.15
Default: ``True``
Whether to enable referer middleware.
@ -352,8 +349,6 @@ Whether to enable referer middleware.
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy

37
docs/topics/spiders.rst
View File

@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods::
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = ['http://www.example.com/categories/%s' % category]
self.start_urls = [f'http://www.example.com/categories/{category}']
# ...
The default `__init__` method will take any spider arguments
@ -292,7 +292,15 @@ The above example can also be written as follows::
name = 'myspider'
def start_requests(self):
yield scrapy.Request('http://www.example.com/categories/%s' % self.category)
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
@ -360,9 +368,10 @@ CrawlSpider
This spider also exposes an overrideable method:
.. method:: parse_start_url(response)
.. method:: parse_start_url(response, **kwargs)
This method is called for the start_urls responses. It allows to parse
This method is called for each response produced for the URLs in
the spider's ``start_urls`` attribute. It allows to parse
the initial responses and must return either an
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
object, or an iterable containing any of them.
@ -388,11 +397,6 @@ Crawling rules
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
callback, since the :class:`CrawlSpider` uses the ``parse`` method
itself to implement its logic. So if you override the ``parse`` method,
the crawl spider will no longer work.
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
callback function.
@ -418,6 +422,11 @@ Crawling rules
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
unexpected behaviour can occur otherwise.
.. versionadded:: 2.0
The *errback* parameter.
@ -451,6 +460,11 @@ Let's now take a look at an example CrawlSpider with rules::
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
url = response.xpath('//td[@id="additional_data"]/@href').get()
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
def parse_additional_page(self, response, item):
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
return item
@ -544,6 +558,11 @@ XMLFeedSpider
those results. It must return a list of results (items or requests).
.. warning:: Because of its internal implementation, you must explicitly set
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
unexpected behaviour can occur otherwise.
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~

3
docs/topics/telnetconsole.rst
View File

@ -110,11 +110,10 @@ using the telnet console::
Execution engine status
time()-engine.start_time : 8.62972998619
engine.has_capacity() : False
len(engine.downloader.active) : 16
engine.scraper.is_idle() : False
engine.spider.name : followall
engine.spider_is_idle(engine.spider) : False
engine.spider_is_idle() : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0

2
docs/utils/linkfix.py
View File

@ -23,7 +23,7 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:

22
docs/versioning.rst
View File

@ -1,7 +1,7 @@
.. _versioning:
============================
Versioning and API Stability
Versioning and API stability
============================
Versioning
@ -34,7 +34,7 @@ For example:
production)
API Stability
API stability
=============
API stability was one of the major goals for the *1.0* release.
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
same way.
.. _deprecation-policy:
Deprecation policy
==================
We aim to maintain support for deprecated Scrapy features for at least 1 year.
For example, if a feature is deprecated in a Scrapy version released on
June 15th 2020, that feature should continue to work in versions released on
June 14th 2021 or before that.
Any new Scrapy release after a year *may* remove support for that deprecated
feature.
All deprecated features removed in a Scrapy release are explicitly mentioned in
the :ref:`release notes <news>`.
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases

2
extras/qps-bench-server.py
View File

@ -37,7 +37,7 @@ class Root(Resource):
if now - self.lastmark >= 3:
self.lastmark = now
qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}')
if 'latency' in request.args:
latency = float(request.args['latency'][0])

6
extras/qpsclient.py
View File

@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
@ -37,11 +37,11 @@ class QPSSpider(Spider):
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += '?latency={0}'.format(self.latency)
url += f'?latency={self.latency}'
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
else:
urls = [url]

3
pylintrc
View File

@ -24,6 +24,7 @@ disable=abstract-method,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
@ -68,6 +69,7 @@ disable=abstract-method,
pointless-statement,
pointless-string-statement,
protected-access,
raise-missing-from,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
@ -75,6 +77,7 @@ disable=abstract-method,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,

24
pytest.ini
View File

@ -1,4 +1,5 @@
[pytest]
xfail_strict = true
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
@ -17,26 +18,7 @@ addopts =
--ignore=docs/topics/stats.rst
--ignore=docs/topics/telnetconsole.rst
--ignore=docs/utils
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-max-line-length = 119
flake8-ignore =
W503
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/http.py F403
scrapy/utils/markup.py F403
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
filterwarnings=
ignore::DeprecationWarning:twisted.web.test.test_webclient

2
scrapy/VERSION
View File

@ -1 +1 @@
2.2.0
2.5.0

6
scrapy/__init__.py
View File

@ -22,14 +22,14 @@ __all__ = [
# Scrapy and Twisted versions
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 5, 2):
print("Scrapy %s requires Python 3.5.2" % __version__)
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
sys.exit(1)

26
scrapy/cmdline.py
View File

@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
# scrapy.utils.spider.iter_spider_classes
for module in walk_modules(module_name):
for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, ScrapyCommand) and \
obj.__module__ == module.__name__ and \
not obj == ScrapyCommand:
if (
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and not obj == ScrapyCommand
):
yield obj
@ -42,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
if inspect.isclass(obj):
cmds[entry_point.name] = obj()
else:
raise Exception("Invalid entry point %s" % entry_point.name)
raise Exception(f"Invalid entry point {entry_point.name}")
return cmds
@ -65,11 +67,11 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
print(f"Scrapy {version} - no active project\n")
def _print_commands(settings, inproject):
@ -79,7 +81,7 @@ def _print_commands(settings, inproject):
print("Available commands:")
cmds = _get_commands_dict(settings, inproject)
for cmdname, cmdclass in sorted(cmds.items()):
print(" %-13s %s" % (cmdname, cmdclass.short_desc()))
print(f" {cmdname:<13} {cmdclass.short_desc()}")
if not inproject:
print()
print(" [ more ] More commands available when run from project directory")
@ -89,7 +91,7 @@ def _print_commands(settings, inproject):
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print(f"Unknown command: {cmdname}\n")
print('Use "scrapy" to see available commands')
@ -131,7 +133,7 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
@ -153,7 +155,7 @@ def _run_command(cmd, args, opts):
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx('cmd.run(args, opts)', globals(), loc)

23
scrapy/commands/__init__.py
View File

@ -3,6 +3,8 @@ Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
from typing import Any, Dict
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@ -15,7 +17,7 @@ class ScrapyCommand:
crawler_process = None
# default settings to be used for this command instead of global defaults
default_settings = {}
default_settings: Dict[str, Any] = {}
exitcode = 0
@ -61,7 +63,7 @@ class ScrapyCommand:
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
@ -108,16 +110,18 @@ class ScrapyCommand:
class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl and runspider commands
Common class used to share functionality between the crawl, parse and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -125,6 +129,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
if opts.output or opts.overwrite_output:
feeds = feed_process_params_from_cli(
self.settings,
opts.output,
opts.output_format,
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')

2
scrapy/commands/bench.py
View File

@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
def start_requests(self):
qargs = {'total': self.total, 'show': self.show}
url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1))
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
return [scrapy.Request(url, dont_filter=True)]
def parse(self, response):

38
scrapy/commands/check.py
View File

@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult):
plural = "s" if run != 1 else ""
writeln(self.separator2)
writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s")
writeln()
infos = []
@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult):
write("FAILED")
failed, errored = map(len, (self.failures, self.errors))
if failed:
infos.append("failures=%d" % failed)
infos.append(f"failures={failed}")
if errored:
infos.append("errors=%d" % errored)
infos.append(f"errors={errored}")
else:
write("OK")
if infos:
writeln(" (%s)" % (", ".join(infos),))
writeln(f" ({', '.join(infos)})")
else:
write("\n")
@ -78,19 +78,19 @@ class Command(ScrapyCommand):
elif tested_methods:
self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(f' * {method}')
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())

8
scrapy/commands/crawl.py
View File

@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
if len(args) < 1:
raise UsageError()
elif len(args) > 1:
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
spname = args[0]
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
else:
self.crawler_process.start()
if self.crawler_process.bootstrap_failed or \
(hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
if (
self.crawler_process.bootstrap_failed
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
):
self.exitcode = 1

4
scrapy/commands/edit.py
View File

@ -32,8 +32,8 @@ class Command(ScrapyCommand):
try:
spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError:
return self._err("Spider not found: %s" % args[0])
return self._err(f"Spider not found: {args[0]}")
sfile = sys.modules[spidercls.__module__].__file__
sfile = sfile.replace('.pyc', '.py')
self.exitcode = os.system('%s "%s"' % (editor, sfile))
self.exitcode = os.system(f'{editor} "{sfile}"')

6
scrapy/commands/fetch.py
View File

@ -19,8 +19,10 @@ class Command(ScrapyCommand):
return "Fetch a URL using the Scrapy downloader"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and print its content " \
"to stdout. You may want to use --nolog to disable logging"
return (
"Fetch a URL using the Scrapy downloader and print its content"
" to stdout. You may want to use --nolog to disable logging"
)
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)

67
scrapy/commands/genspider.py
View File

@ -66,31 +66,25 @@ class Command(ScrapyCommand):
print("Cannot create a spider with the same name as your project")
return
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider already exists and not --force then halt
if not opts.force:
print("Spider %r already exists in module:" % name)
print(" %s" % spidercls.__module__)
return
if not opts.force and self._spider_exists(name):
return
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
if opts.edit:
self.exitcode = os.system('scrapy edit "%s"' % name)
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = ''.join(s.capitalize() for s in module.split('_'))
tvars = {
'project_name': self.settings.get('BOT_NAME'),
'ProjectName': string_camelcase(self.settings.get('BOT_NAME')),
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
'classname': f'{capitalized_module}Spider'
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -98,29 +92,58 @@ class Command(ScrapyCommand):
else:
spiders_module = None
spiders_dir = "."
spider_file = "%s.py" % join(spiders_dir, module)
spider_file = f"{join(spiders_dir, module)}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
print(f"Created spider {name!r} using template {template_name!r} ",
end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, '%s.tmpl' % template)
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
return template_file
print("Unable to find template: %s\n" % template)
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(" %s" % splitext(filename)[0])
print(f" {splitext(filename)[0]}")
def _spider_exists(self, name):
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
return True
return False
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
pass
else:
# if spider with same name exists
print(f"Spider {name!r} already exists in module:")
print(f" {spidercls.__module__}")
return True
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
return True
return False
@property
def templates_dir(self):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'spiders')
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'spiders'
)

34
scrapy/commands/parse.py
View File

@ -1,26 +1,26 @@
import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = True
spider = None
items = {}
requests = {}
items: Dict[int, list] = {}
requests: Dict[int, list] = {}
first_response = None
@ -31,11 +31,9 @@ class Command(ScrapyCommand):
return "Parse URL (using its spider) and print the results"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
@ -100,13 +98,13 @@ class Command(ScrapyCommand):
if opts.verbose:
for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
print(f'\n>>> DEPTH LEVEL: {level} <<<')
if not opts.noitems:
self.print_items(level, colour)
if not opts.nolinks:
self.print_requests(level, colour)
else:
print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
if not opts.noitems:
self.print_items(colour=colour)
if not opts.nolinks:
@ -200,12 +198,15 @@ class Command(ScrapyCommand):
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
return requests
scraped_data += requests
return scraped_data
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
@ -221,18 +222,11 @@ class Command(ScrapyCommand):
return request
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
BaseRunSpiderCommand.process_options(self, args, opts)
self.process_spider_arguments(opts)
self.process_request_meta(opts)
self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
if opts.meta:
try:

10
scrapy/commands/runspider.py
View File

@ -11,8 +11,8 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
raise ValueError("Not a Python source file: %s" % abspath)
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand):
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
raise UsageError("File not found: %s\n" % filename)
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError("Unable to load %r: %s\n" % (filename, e))
raise UsageError(f"Unable to load {filename!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError("No spider found in file: %s\n" % filename)
raise UsageError(f"No spider found in file: {filename}\n")
spidercls = spclasses.pop()
self.crawler_process.crawl(spidercls, **opts.spargs)

54
scrapy/commands/startproject.py
View File

@ -1,10 +1,10 @@
import re
import os
import stat
import string
from importlib import import_module
from os.path import join, exists, abspath
from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
('${project_name}', 'middlewares.py.tmpl'),
)
IGNORE = ignore_patterns('*.pyc', '.svn')
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
def _make_writable(path):
current_permissions = os.stat(path).st_mode
os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
class Command(ScrapyCommand):
@ -47,7 +52,7 @@ class Command(ScrapyCommand):
print('Error: Project names must begin with a letter and contain'
' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
print(f'Error: Module {project_name!r} already exists')
else:
return True
return False
@ -78,30 +83,10 @@ class Command(ScrapyCommand):
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
_make_writable(dstname)
copystat(src, dst)
self._set_rw_permissions(dst)
def _set_rw_permissions(self, path):
"""
Sets permissions of a directory tree to +rw and +rwx for folders.
This is necessary if the start template files come without write
permissions.
"""
mode_rw = (stat.S_IRUSR
| stat.S_IWUSR
| stat.S_IRGRP
| stat.S_IROTH)
mode_x = (stat.S_IXUSR
| stat.S_IXGRP
| stat.S_IXOTH)
os.chmod(path, mode_rw | mode_x)
for root, dirs, files in os.walk(path):
for dir in dirs:
os.chmod(join(root, dir), mode_rw | mode_x)
for file in files:
os.chmod(join(root, file), mode_rw)
_make_writable(dst)
def run(self, args, opts):
if len(args) not in (1, 2):
@ -115,7 +100,7 @@ class Command(ScrapyCommand):
if exists(join(project_dir, 'scrapy.cfg')):
self.exitcode = 1
print('Error: scrapy.cfg already exists in %s' % abspath(project_dir))
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
return
if not self._is_valid_name(project_name):
@ -128,15 +113,16 @@ class Command(ScrapyCommand):
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
print("You can start your first spider with:")
print(" cd %s" % project_dir)
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
'project'
)

5
scrapy/commands/version.py
View File

@ -23,8 +23,7 @@ class Command(ScrapyCommand):
if opts.verbose:
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)
patt = "%-{}s : %s".format(width)
for name, version in versions:
print(patt % (name, version))
print(f"{name:<{width}} : {version}")
else:
print("Scrapy %s" % scrapy.__version__)
print(f"Scrapy {scrapy.__version__}")

5
scrapy/commands/view.py
View File

@ -8,11 +8,10 @@ class Command(fetch.Command):
return "Open URL in browser, as seen by Scrapy"
def long_desc(self):
return "Fetch a URL using the Scrapy downloader and show its " \
"contents in a browser"
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
def add_options(self, parser):
super(Command, self).add_options(parser)
super().add_options(parser)
parser.remove_option("--headers")
def _print_response(self, response, opts):

131
scrapy/contracts/__init__.py
View File

@ -1,16 +1,77 @@
import sys
import re
import sys
from functools import wraps
from inspect import getmembers
from typing import Dict
from unittest import TestCase
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
from scrapy.utils.spider import iterate_spider_output
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
class ContractsManager:
contracts = {}
contracts: Dict[str, Contract] = {}
def __init__(self, contracts):
for contract in contracts:
@ -107,73 +168,13 @@ class ContractsManager:
request.errback = eb_wrapper
class Contract:
""" Abstract class for contracts """
request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name)
self.args = args
def add_pre_hook(self, request, results):
if hasattr(self, 'pre_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
results.stopTest(self.testcase_pre)
except AssertionError:
results.addFailure(self.testcase_pre, sys.exc_info())
except Exception:
results.addError(self.testcase_pre, sys.exc_info())
else:
results.addSuccess(self.testcase_pre)
finally:
return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
return request
def add_post_hook(self, request, results):
if hasattr(self, 'post_process'):
cb = request.callback
@wraps(cb)
def wrapper(response, **cb_kwargs):
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
results.stopTest(self.testcase_post)
except AssertionError:
results.addFailure(self.testcase_post, sys.exc_info())
except Exception:
results.addError(self.testcase_post, sys.exc_info())
else:
results.addSuccess(self.testcase_post)
finally:
return output
request.callback = wrapper
return request
def adjust_request_args(self, args):
return args
def _create_testcase(method, desc):
spider = method.__self__.name
class ContractTestCase(TestCase):
def __str__(_self):
return "[%s] %s (%s)" % (spider, method.__name__, desc)
return f"[{spider}] {method.__name__} ({desc})"
name = '%s_%s' % (spider, method.__name__)
name = f'{spider}_{method.__name__}'
setattr(ContractTestCase, name, lambda x: x)
return ContractTestCase(name)

14
scrapy/contracts/default.py
View File

@ -56,12 +56,11 @@ class ReturnsContract(Contract):
}
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
)
self.obj_name = self.args[0] or None
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
@ -88,10 +87,9 @@ class ReturnsContract(Contract):
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
expected = f'{self.min_bound}..{self.max_bound}'
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}")
class ScrapesContract(Contract):
@ -106,5 +104,5 @@ class ScrapesContract(Contract):
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
missing_str = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str)
missing_fields = ", ".join(missing)
raise ContractFail(f"Missing fields: {missing_fields}")

16
scrapy/core/downloader/__init__.py
View File

@ -41,17 +41,17 @@ class Slot:
def __repr__(self):
cls_name = self.__class__.__name__
return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % (
cls_name, self.concurrency, self.delay, self.randomize_delay)
return (f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})")
def __str__(self):
return (
"<downloader.Slot concurrency=%r delay=%0.2f randomize_delay=%r "
"len(active)=%d len(queue)=%d len(transferring)=%d lastseen=%s>" % (
self.concurrency, self.delay, self.randomize_delay,
len(self.active), len(self.queue), len(self.transferring),
datetime.fromtimestamp(self.lastseen).isoformat()
)
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
)

59
scrapy/core/downloader/contextfactory.py
View File

@ -1,10 +1,15 @@
import warnings
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.utils.misc import create_instance, load_object
@implementer(IPolicyForHTTPS)
@ -20,7 +25,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
@ -45,7 +50,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
@ -81,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
The default OpenSSL method is ``TLS_METHOD`` (also called
``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname, port):
def creatorForNetloc(self, hostname, port):
# trustRoot set to platformTrust() will use the platform's root CAs.
#
# This means that a website like https://www.cacert.org will be rejected
@ -92,3 +97,49 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)
@implementer(IPolicyForHTTPS)
class AcceptableProtocolsContextFactory:
"""Context factory to used to override the acceptable protocols
to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
negotiation.
"""
def __init__(self, context_factory, acceptable_protocols):
verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory = context_factory
self._acceptable_protocols = acceptable_protocols
def creatorForNetloc(self, hostname, port):
options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
return options
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
return context_factory

3
scrapy/core/downloader/handlers/__init__.py
View File

@ -71,8 +71,7 @@ class DownloadHandlers:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported("Unsupported URL scheme '%s': %s" %
(scheme, self._notconfigured[scheme]))
raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}")
return handler.download_request(request, spider)
@defer.inlineCallbacks

125
scrapy/core/downloader/handlers/http11.py
View File

@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
method=self._sslMethod,
)
except TypeError:
# use context factory defaults
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
@ -121,12 +98,13 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
_responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
_truncatedLength = 1000
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@ -167,10 +145,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
else:
extra = rcvd_bytes[:32]
extra = rcvd_bytes[:self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
self._host, self._port, extra)))
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')
)
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@ -178,7 +157,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@ -215,7 +194,7 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -235,7 +214,7 @@ class TunnelingAgent(Agent):
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
return super(TunnelingAgent, self)._requestWithEndpoint(
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
@ -249,7 +228,7 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
super(ScrapyProxyAgent, self).__init__(
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
@ -302,11 +281,14 @@ class ScrapyAgent:
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
warnings.warn(
"Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Zyte Smart Proxy Manager, it doesn't require "
"this mode anymore, so you should update scrapy-crawlera "
"to scrapy-zyte-smartproxy and remove '?noconnect' "
"from the Zyte Smart Proxy Manager URL.",
ScrapyDeprecationWarning,
)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
@ -375,13 +357,44 @@ class ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result, request, start_time):
request.meta['download_latency'] = time() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response):
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b'Content-Length'] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
def _cb_bodyready(self, txresponse, request):
headers_received_result = self._crawler.signals.send_catch_log(
signal=signals.headers_received,
headers=self._headers_from_twisted_response(txresponse),
body_length=txresponse.length,
request=request,
spider=self._crawler.spider,
)
for handler, result in headers_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__})
txresponse._transport.stopProducing()
with suppress(AttributeError):
txresponse._transport._producer.loseConnection()
return {
"txresponse": txresponse,
"body": b"",
"flags": ["download_stopped"],
"certificate": None,
"ip_address": None,
"failure": result if result.value.fail else None,
}
# deliverBody hangs for responses without body
if txresponse.length == 0:
return {
@ -398,13 +411,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
error_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
logger.warning(warning_msg, warning_args)
logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
raise defer.CancelledError(error_msg % error_args)
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@ -434,8 +448,13 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
headers = self._headers_from_twisted_response(result["txresponse"])
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
try:
version = result["txresponse"].version
protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
except (AttributeError, TypeError, IndexError):
protocol = None
response = respcls(
url=url,
status=int(result["txresponse"].code),
@ -444,6 +463,7 @@ class ScrapyAgent:
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
protocol=protocol,
)
if result.get("failure"):
result["failure"].value.response = response
@ -522,16 +542,17 @@ class _ResponseReader(protocol.Protocol):
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
self.transport.stopProducing()
self.transport._producer.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()

129
scrapy/core/downloader/handlers/http2.py Normal file
View File

@ -0,0 +1,129 @@
import warnings
from time import time
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
from twisted.internet.base import DelayedCall
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
class H2DownloadHandler:
def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
self._crawler = crawler
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
self._context_factory = load_context_factory_from_settings(settings, crawler)
@classmethod
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider: Spider) -> Deferred:
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
crawler=self._crawler,
)
return agent.download_request(request, spider)
def close(self) -> None:
self._pool.close_connections()
class ScrapyH2Agent:
_Agent = H2Agent
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self, context_factory,
pool: H2ConnectionPool,
connect_timeout: int = 10,
bind_address: Optional[bytes] = None,
crawler: Optional[Crawler] = None,
) -> None:
self._context_factory = context_factory
self._connect_timeout = connect_timeout
self._bind_address = bind_address
self._pool = pool
self._crawler = crawler
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
bind_address = request.meta.get('bindaddress') or self._bind_address
proxy = request.meta.get('proxy')
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is not "
"supported by the downloader handler. If you use Zyte "
"Smart Proxy Manager, it doesn't require this mode "
"anymore, so you should update scrapy-crawlera to "
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
"Zyte Smart Proxy Manager URL."
)
if scheme == b'https' and not omit_connect_tunnel:
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
context_factory=self._context_factory,
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool,
)
def download_request(self, request: Request, spider: Spider) -> Deferred:
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connect_timeout
agent = self._get_agent(request, timeout)
start_time = time()
d = agent.request(request, spider)
d.addCallback(self._cb_latency, request, start_time)
timeout_cl = reactor.callLater(timeout, d.cancel)
d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
return d
@staticmethod
def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
request.meta['download_latency'] = time() - start_time
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
if timeout_cl.active():
timeout_cl.cancel()
return response
url = urldefrag(request.url)[0]
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")

59
scrapy/core/downloader/handlers/s3.py
View File

@ -2,41 +2,20 @@ from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
def _get_boto_connection():
from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, method, bucket, key, headers, *args, **kwargs):
return headers
class _v20_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, http_request, *args, **kwargs):
http_request.authorize(connection=self)
return http_request.headers
try:
import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
_S3Connection = _v20_S3Connection
return _S3Connection
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
@ -51,23 +30,15 @@ class S3DownloadHandler:
self.anon = kw.get('anon')
self._signer = None
if is_botocore():
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
_http_handler = create_instance(
objcls=httpdownloadhandler,
@ -85,14 +56,14 @@ class S3DownloadHandler:
scheme = 'https' if request.meta.get('is_secure') else 'http'
bucket = p.hostname
path = p.path + '?' + p.query if p.query else p.path
url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
url=f'{scheme}://s3.amazonaws.com/{bucket}{path}',
headers=request.headers.to_unicode_dict(),
data=request.body)
self._signer.add_auth(awsrequest)

26
scrapy/core/downloader/middleware.py
View File

@ -3,8 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from twisted.internet import defer
from typing import Callable, Union
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
@ -29,22 +33,22 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
def download(self, download_func: Callable, request: Request, spider: Spider):
@defer.inlineCallbacks
def process_request(request):
def process_request(request: Request):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
return response
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
def process_response(response):
def process_response(response: Union[Response, Request]):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
@ -54,22 +58,22 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__qualname__} must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
return response
return response
@defer.inlineCallbacks
def process_exception(failure):
def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"
)
if response:
return response

29
scrapy/core/downloader/tls.py
View File

@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@ -28,13 +27,6 @@ openssl_methods = {
}
if twisted_version < (17, 0, 0):
from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
else:
def set_tlsext_host_name(connection, hostNameBytes):
connection.set_tlsext_host_name(hostNameBytes)
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
@ -47,26 +39,19 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
"""
def __init__(self, hostname, ctx, verbose_logging=False):
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
super().__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL.SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
else:
logger.debug('SSL connection to %s using cipher %s',
self._hostnameASCII,
connection.get_cipher_name(),
)
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),

100
scrapy/core/downloader/webclient.py
View File

@ -2,13 +2,13 @@ import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet import defer, reactor
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes
@ -91,22 +91,38 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds."
% (self.factory.url, self.factory.timeout)))
defer.TimeoutError(f"Getting {self.factory.url} took longer "
f"than {self.factory.timeout} seconds."))
class ScrapyHTTPClientFactory(HTTPClientFactory):
"""Scrapy implementation of the HTTPClientFactory overwriting the
setUrl method to make use of our Url object that cache the parse
result.
"""
# This class used to inherit from Twisted’s
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overriden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
@ -141,21 +157,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
elif self.method == b'POST':
self.headers['Content-Length'] = 0
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
return respcls(url=self._url, status=status, headers=headers, body=body)
def __repr__(self):
return f"<{self.__class__.__name__}: {self.url}>"
def _set_connection_attributes(self, request):
parsed = urlparse_cached(request)
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
proxy = request.meta.get('proxy')
if proxy:
self.scheme, _, self.host, self.port, _ = _parse(proxy)
self.path = self.url
def _cancelTimeout(self, result, timeoutCall):
if timeoutCall.active():
timeoutCall.cancel()
return result
def buildProtocol(self, addr):
p = ClientFactory.buildProtocol(self, addr)
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
def gotStatus(self, version, status, message):
"""
Set the status of the request on us.
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}
"""
self.version, self.status, self.message = version, status, message
def page(self, page):
if self.waiting:
self.waiting = 0
self.deferred.callback(page)
def noPage(self, reason):
if self.waiting:
self.waiting = 0
self.deferred.errback(reason)
def clientConnectionFailed(self, _, reason):
"""
When a connection attempt fails, the request cannot be issued. If no
result has yet been provided to the result Deferred, provide the
connection failure reason as an error result.
"""
if self.waiting:
self.waiting = 0
# If the connection attempt failed, there is nothing more to
# disconnect, so just fire that Deferred now.
self._disconnectedDeferred.callback(None)
self.deferred.errback(reason)

449
scrapy/core/engine.py
View File

@ -1,51 +1,62 @@
"""
This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
For more information see docs/topics/architecture.rst
"""
import logging
import warnings
from time import time
from typing import Callable, Iterable, Iterator, Optional, Set, Union
from twisted.internet import defer, task
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.scraper import Scraper
from scrapy.exceptions import DontCloseSpider
from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Response, Request
from scrapy.utils.misc import load_object
from scrapy.utils.reactor import CallLaterOnce
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.reactor import CallLaterOnce
logger = logging.getLogger(__name__)
class Slot:
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
def __init__(
self,
start_requests: Iterable,
close_if_idle: bool,
nextcall: CallLaterOnce,
scheduler,
) -> None:
self.closing: Optional[Deferred] = None
self.inprogress: Set[Request] = set()
self.start_requests: Optional[Iterator] = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
self.scheduler = scheduler
self.heartbeat = task.LoopingCall(nextcall.schedule)
self.heartbeat = LoopingCall(nextcall.schedule)
def add_request(self, request):
def add_request(self, request: Request) -> None:
self.inprogress.add(request)
def remove_request(self, request):
def remove_request(self, request: Request) -> None:
self.inprogress.remove(request)
self._maybe_fire_closing()
def close(self):
self.closing = defer.Deferred()
def close(self) -> Deferred:
self.closing = Deferred()
self._maybe_fire_closing()
return self.closing
def _maybe_fire_closing(self):
if self.closing and not self.inprogress:
def _maybe_fire_closing(self) -> None:
if self.closing is not None and not self.inprogress:
if self.nextcall:
self.nextcall.cancel()
if self.heartbeat.running:
@ -54,203 +65,236 @@ class Slot:
class ExecutionEngine:
def __init__(self, crawler, spider_closed_callback):
def __init__(self, crawler, spider_closed_callback: Callable) -> None:
self.crawler = crawler
self.settings = crawler.settings
self.signals = crawler.signals
self.logformatter = crawler.logformatter
self.slot = None
self.spider = None
self.slot: Optional[Slot] = None
self.spider: Optional[Spider] = None
self.running = False
self.paused = False
self.scheduler_cls = load_object(self.settings['SCHEDULER'])
self.scheduler_cls = self._get_scheduler_class(crawler.settings)
downloader_cls = load_object(self.settings['DOWNLOADER'])
self.downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
@defer.inlineCallbacks
def start(self):
"""Start the execution engine"""
def _get_scheduler_class(self, settings: BaseSettings) -> type:
from scrapy.core.scheduler import BaseScheduler
scheduler_cls = load_object(settings["SCHEDULER"])
if not issubclass(scheduler_cls, BaseScheduler):
raise TypeError(
f"The provided scheduler class ({settings['SCHEDULER']})"
" does not fully implement the scheduler interface"
)
return scheduler_cls
@inlineCallbacks
def start(self) -> Deferred:
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
self._closewait = defer.Deferred()
self._closewait = Deferred()
yield self._closewait
def stop(self):
"""Stop the execution engine gracefully"""
def stop(self) -> Deferred:
"""Gracefully stop the execution engine"""
@inlineCallbacks
def _finish_stopping_engine(_) -> Deferred:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
if not self.running:
raise RuntimeError("Engine not running")
self.running = False
dfd = self._close_all_spiders()
return dfd.addBoth(lambda _: self._finish_stopping_engine())
dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
return dfd.addBoth(_finish_stopping_engine)
def close(self):
"""Close the execution engine gracefully.
If it has already been started, stop it. In all cases, close all spiders
and the downloader.
def close(self) -> Deferred:
"""
Gracefully close the execution engine.
If it has already been started, stop it. In all cases, close the spider and the downloader.
"""
if self.running:
# Will also close spiders and downloader
return self.stop()
elif self.open_spiders:
# Will also close downloader
return self._close_all_spiders()
else:
return defer.succeed(self.downloader.close())
return self.stop() # will also close spider and downloader
if self.spider is not None:
return self.close_spider(self.spider, reason="shutdown") # will also close downloader
return succeed(self.downloader.close())
def pause(self):
"""Pause the execution engine"""
def pause(self) -> None:
self.paused = True
def unpause(self):
"""Resume the execution engine"""
def unpause(self) -> None:
self.paused = False
def _next_request(self, spider):
slot = self.slot
if not slot:
return
def _next_request(self) -> None:
assert self.slot is not None # typing
assert self.spider is not None # typing
if self.paused:
return
return None
while not self._needs_backout(spider):
if not self._next_request_from_scheduler(spider):
break
while not self._needs_backout() and self._next_request_from_scheduler() is not None:
pass
if slot.start_requests and not self._needs_backout(spider):
if self.slot.start_requests is not None and not self._needs_backout():
try:
request = next(slot.start_requests)
request = next(self.slot.start_requests)
except StopIteration:
slot.start_requests = None
self.slot.start_requests = None
except Exception:
slot.start_requests = None
logger.error('Error while obtaining start requests',
exc_info=True, extra={'spider': spider})
self.slot.start_requests = None
logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
else:
self.crawl(request, spider)
self.crawl(request)
if self.spider_is_idle(spider) and slot.close_if_idle:
self._spider_idle(spider)
if self.spider_is_idle() and self.slot.close_if_idle:
self._spider_idle()
def _needs_backout(self, spider):
slot = self.slot
return not self.running \
or slot.closing \
or self.downloader.needs_backout() \
or self.scraper.slot.needs_backout()
def _needs_backout(self) -> bool:
return (
not self.running
or self.slot.closing # type: ignore[union-attr]
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout() # type: ignore[union-attr]
)
def _next_request_from_scheduler(self, spider):
slot = self.slot
request = slot.scheduler.next_request()
if not request:
return
d = self._download(request, spider)
d.addBoth(self._handle_downloader_output, request, spider)
def _next_request_from_scheduler(self) -> Optional[Deferred]:
assert self.slot is not None # typing
assert self.spider is not None # typing
request = self.slot.scheduler.next_request()
if request is None:
return None
d = self._download(request, self.spider)
d.addBoth(self._handle_downloader_output, request)
d.addErrback(lambda f: logger.info('Error while handling downloader output',
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
d.addBoth(lambda _: slot.remove_request(request))
extra={'spider': self.spider}))
d.addBoth(lambda _: self.slot.remove_request(request))
d.addErrback(lambda f: logger.info('Error while removing request from slot',
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
d.addBoth(lambda _: slot.nextcall.schedule())
extra={'spider': self.spider}))
d.addBoth(lambda _: self.slot.nextcall.schedule())
d.addErrback(lambda f: logger.info('Error while scheduling new request',
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
extra={'spider': self.spider}))
return d
def _handle_downloader_output(self, response, request, spider):
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
)
def _handle_downloader_output(
self, result: Union[Request, Response, Failure], request: Request
) -> Optional[Deferred]:
assert self.spider is not None # typing
if not isinstance(result, (Request, Response, Failure)):
raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
self.crawl(response, spider)
return
# response is a Response or Failure
d = self.scraper.enqueue_scrape(response, request, spider)
d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
if isinstance(result, Request):
self.crawl(result)
return None
d = self.scraper.enqueue_scrape(result, request, self.spider)
d.addErrback(
lambda f: logger.error(
"Error while enqueuing downloader output",
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider},
)
)
return d
def spider_is_idle(self, spider):
if not self.scraper.slot.is_idle():
# scraper is not idle
def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
return False
if self.downloader.active:
# downloader has pending requests
if self.downloader.active: # downloader has pending requests
return False
if self.slot.start_requests is not None:
# not all start requests are handled
if self.slot.start_requests is not None: # not all start requests are handled
return False
if self.slot.scheduler.has_pending_requests():
# scheduler has pending requests
return False
return True
@property
def open_spiders(self):
return [self.spider] if self.spider else []
def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
self._schedule_request(request, self.spider)
self.slot.nextcall.schedule() # type: ignore[union-attr]
def has_capacity(self):
"""Does the engine have capacity to handle more spiders"""
return not bool(self.slot)
def crawl(self, request, spider):
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
self.schedule(request, spider)
self.slot.nextcall.schedule()
def schedule(self, request, spider):
def _schedule_request(self, request: Request, spider: Spider) -> None:
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
if not self.slot.scheduler.enqueue_request(request):
if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
def download(self, request, spider):
d = self._download(request, spider)
d.addBoth(self._downloaded, self.slot, request, spider)
return d
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
if spider is None:
spider = self.spider
else:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if spider is not self.spider:
logger.warning("The spider '%s' does not match the open spider", spider.name)
if spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request, spider).addBoth(self._downloaded, request, spider)
def _downloaded(self, response, slot, request, spider):
slot.remove_request(request)
return self.download(response, spider) if isinstance(response, Request) else response
def _downloaded(
self, result: Union[Response, Request], request: Request, spider: Spider
) -> Union[Deferred, Response]:
assert self.slot is not None # typing
self.slot.remove_request(request)
return self.download(result, spider) if isinstance(result, Request) else result
def _download(self, request, spider):
slot = self.slot
slot.add_request(request)
def _download(self, request: Request, spider: Spider) -> Deferred:
assert self.slot is not None # typing
def _on_success(response):
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
self.slot.add_request(request)
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
if not isinstance(result, (Response, Request)):
raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
if isinstance(result, Response):
if result.request is None:
result.request = request
logkws = self.logformatter.crawled(result.request, result, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
self.signals.send_catch_log(signals.response_received,
response=response, request=request, spider=spider)
return response
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
self.signals.send_catch_log(
signal=signals.response_received,
response=result,
request=result.request,
spider=spider,
)
return result
def _on_complete(_):
slot.nextcall.schedule()
self.slot.nextcall.schedule()
return _
dwld = self.downloader.fetch(request, spider)
@ -258,58 +302,53 @@ class ExecutionEngine:
dwld.addBoth(_on_complete)
return dwld
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
@inlineCallbacks
def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
if self.slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)
nextcall = CallLaterOnce(self._next_request)
scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.slot = slot
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
yield scheduler.open(spider)
if hasattr(scheduler, "open"):
yield scheduler.open(spider)
yield self.scraper.open_spider(spider)
self.crawler.stats.open_spider(spider)
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
slot.nextcall.schedule()
slot.heartbeat.start(5)
self.slot.nextcall.schedule()
self.slot.heartbeat.start(5)
def _spider_idle(self, spider):
"""Called when a spider gets idle. This function is called when there
are no remaining pages to download or schedule. It can be called
multiple times. If some extension raises a DontCloseSpider exception
(in the spider_idle signal handler) the spider is not closed until the
next loop and this function is guaranteed to be called (at least) once
again for this spider.
def _spider_idle(self) -> None:
"""
res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
exception, the spider is not closed until the next loop and this function is guaranteed to be called
(at least) once again.
"""
assert self.spider is not None # typing
res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return
return None
if self.spider_is_idle():
self.close_spider(self.spider, reason='finished')
if self.spider_is_idle(spider):
self.close_spider(spider, reason='finished')
def close_spider(self, spider, reason='cancelled'):
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
"""Close (cancel) spider and clear all its outstanding requests"""
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
slot = self.slot
if slot.closing:
return slot.closing
logger.info("Closing spider (%(reason)s)",
{'reason': reason},
extra={'spider': spider})
if self.slot.closing is not None:
return self.slot.closing
dfd = slot.close()
logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
def log_failure(msg):
def errback(failure):
logger.error(
msg,
exc_info=failure_to_exc_info(failure),
extra={'spider': spider}
)
dfd = self.slot.close()
def log_failure(msg: str) -> Callable:
def errback(failure: Failure) -> None:
logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
return errback
dfd.addBoth(lambda _: self.downloader.close())
@ -318,19 +357,19 @@ class ExecutionEngine:
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
dfd.addErrback(log_failure('Scraper close failure'))
dfd.addBoth(lambda _: slot.scheduler.close(reason))
dfd.addErrback(log_failure('Scheduler close failure'))
if hasattr(self.slot.scheduler, "close"):
dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
signal=signals.spider_closed, spider=spider, reason=reason))
signal=signals.spider_closed, spider=spider, reason=reason,
))
dfd.addErrback(log_failure('Error while sending spider_close signal'))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
dfd.addErrback(log_failure('Stats close failure'))
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
{'reason': reason},
extra={'spider': spider}))
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
dfd.addBoth(lambda _: setattr(self, 'slot', None))
dfd.addErrback(log_failure('Error while unassigning slot'))
@ -342,12 +381,26 @@ class ExecutionEngine:
return dfd
def _close_all_spiders(self):
dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
dlist = defer.DeferredList(dfds)
return dlist
@property
def open_spiders(self) -> list:
warnings.warn(
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return [self.spider] if self.spider is not None else []
@defer.inlineCallbacks
def _finish_stopping_engine(self):
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
def has_capacity(self) -> bool:
warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
return not bool(self.slot)
def schedule(self, request: Request, spider: Spider) -> None:
warnings.warn(
"ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if self.slot is None:
raise RuntimeError("Engine slot not assigned")
self._schedule_request(request, spider)

0
scrapy/core/http2/__init__.py Normal file
View File

157
scrapy/core/http2/agent.py Normal file
View File

@ -0,0 +1,157 @@
from collections import deque
from typing import Deque, Dict, List, Optional, Tuple
from twisted.internet import defer
from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.endpoints import HostnameEndpoint
from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
class H2ConnectionPool:
def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
self._reactor = reactor
self.settings = settings
# Store a dictionary which is used to get the respective
# H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port)
self._connections: Dict[Tuple, H2ClientProtocol] = {}
# Save all requests that arrive before the connection is established
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
if key in self._pending_requests:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
# instance
d = Deferred()
self._pending_requests[key].append(d)
return d
# Check if we already have a connection to the remote
conn = self._connections.get(key, None)
if conn:
# Return this connection instance wrapped inside a deferred
return defer.succeed(conn)
# No connection is established for the given URI
return self._new_connection(key, uri, endpoint)
def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
self._pending_requests[key] = deque()
conn_lost_deferred = Deferred()
conn_lost_deferred.addCallback(self._remove_connection, key)
factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
conn_d = endpoint.connect(factory)
conn_d.addCallback(self.put_connection, key)
d = Deferred()
self._pending_requests[key].append(d)
return d
def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
self._connections[key] = conn
# Now as we have established a proper HTTP/2 connection
# we fire all the deferred's with the connection instance
pending_requests = self._pending_requests.pop(key, None)
while pending_requests:
d = pending_requests.popleft()
d.callback(conn)
return conn
def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
self._connections.pop(key)
# Call the errback of all the pending requests for this connection
pending_requests = self._pending_requests.pop(key, None)
while pending_requests:
d = pending_requests.popleft()
d.errback(errors)
def close_connections(self) -> None:
"""Close all the HTTP/2 connections and remove them from pool
Returns:
Deferred that fires when all connections have been closed
"""
for conn in self._connections.values():
conn.transport.abortConnection()
class H2Agent:
def __init__(
self,
reactor: ReactorBase,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
self._reactor = reactor
self._pool = pool
self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
self.endpoint_factory = _StandardEndpointFactory(
self._reactor, self._context_factory, connect_timeout, bind_address
)
def get_endpoint(self, uri: URI):
return self.endpoint_factory.endpointForURI(uri)
def get_key(self, uri: URI) -> Tuple:
"""
Arguments:
uri - URI obtained directly from request URL
"""
return uri.scheme, uri.host, uri.port
def request(self, request: Request, spider: Spider) -> Deferred:
uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
try:
endpoint = self.get_endpoint(uri)
except SchemeNotSupported:
return defer.fail(Failure())
key = self.get_key(uri)
d = self._pool.get_connection(key, uri, endpoint)
d.addCallback(lambda conn: conn.request(request, spider))
return d
class ScrapyProxyH2Agent(H2Agent):
def __init__(
self,
reactor: ReactorBase,
proxy_uri: URI,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,
connect_timeout=connect_timeout,
bind_address=bind_address,
)
self._proxy_uri = proxy_uri
def get_endpoint(self, uri: URI):
return self.endpoint_factory.endpointForURI(self._proxy_uri)
def get_key(self, uri: URI) -> Tuple:
"""We use the proxy uri instead of uri obtained from request url"""
return "http-proxy", self._proxy_uri.host, self._proxy_uri.port

418
scrapy/core/http2/protocol.py Normal file
View File

@ -0,0 +1,418 @@
import ipaddress
import itertools
import logging
from collections import deque
from ipaddress import IPv4Address, IPv6Address
from typing import Dict, List, Optional, Union
from h2.config import H2Configuration
from h2.connection import H2Connection
from h2.errors import ErrorCodes
from h2.events import (
Event, ConnectionTerminated, DataReceived, ResponseReceived,
SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
WindowUpdated
)
from h2.exceptions import FrameTooLargeError, H2Error
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
from twisted.internet.protocol import connectionDone, Factory, Protocol
from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from twisted.python.failure import Failure
from twisted.web.client import URI
from zope.interface import implementer
from scrapy.core.http2.stream import Stream, StreamCloseReason
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)
PROTOCOL_NAME = b"h2"
class InvalidNegotiatedProtocol(H2Error):
def __init__(self, negotiated_protocol: bytes) -> None:
self.negotiated_protocol = negotiated_protocol
def __str__(self) -> str:
return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
class RemoteTerminatedConnection(H2Error):
def __init__(
self,
remote_ip_address: Optional[Union[IPv4Address, IPv6Address]],
event: ConnectionTerminated,
) -> None:
self.remote_ip_address = remote_ip_address
self.terminate_event = event
def __str__(self) -> str:
return f'Received GOAWAY frame from {self.remote_ip_address!r}'
class MethodNotAllowed405(H2Error):
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
self.remote_ip_address = remote_ip_address
def __str__(self) -> str:
return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
@implementer(IHandshakeListener)
class H2ClientProtocol(Protocol, TimeoutMixin):
IDLE_TIMEOUT = 240
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
"""
Arguments:
uri -- URI of the base url to which HTTP/2 Connection will be made.
uri is used to verify that incoming client requests have correct
base URL.
settings -- Scrapy project settings
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
that connection was lost
"""
self._conn_lost_deferred = conn_lost_deferred
config = H2Configuration(client_side=True, header_encoding='utf-8')
self.conn = H2Connection(config=config)
# ID of the next request stream
# Following the convention - 'Streams initiated by a client MUST
# use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1)
self._stream_id_generator = itertools.count(start=1, step=2)
# Streams are stored in a dictionary keyed off their stream IDs
self.streams: Dict[int, Stream] = {}
# If requests are received before connection is made we keep
# all requests in a pool and send them as the connection is made
self._pending_request_stream_pool: deque = deque()
# Save an instance of errors raised which lead to losing the connection
# We pass these instances to the streams ResponseFailed() failure
self._conn_lost_errors: List[BaseException] = []
# Some meta data of this connection
# initialized when connection is successfully made
self.metadata: Dict = {
# Peer certificate instance
'certificate': None,
# Address of the server we are connected to which
# is updated when HTTP/2 connection is made successfully
'ip_address': None,
# URI of the peer HTTP/2 connection is made
'uri': uri,
# Both ip_address and uri are used by the Stream before
# initiating the request to verify that the base address
# Variables taken from Project Settings
'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
# Counter to keep track of opened streams. This counter
# is used to make sure that not more than MAX_CONCURRENT_STREAMS
# streams are opened which leads to ProtocolError
# We use simple FIFO policy to handle pending requests
'active_streams': 0,
# Flag to keep track if settings were acknowledged by the remote
# This ensures that we have established a HTTP/2 connection
'settings_acknowledged': False,
}
@property
def h2_connected(self) -> bool:
"""Boolean to keep track of the connection status.
This is used while initiating pending streams to make sure
that we initiate stream only during active HTTP/2 Connection
"""
return bool(self.transport.connected) and self.metadata['settings_acknowledged']
@property
def allowed_max_concurrent_streams(self) -> int:
"""We keep total two streams for client (sending data) and
server side (receiving data) for a single request. To be safe
we choose the minimum. Since this value can change in event
RemoteSettingsChanged we make variable a property.
"""
return min(
self.conn.local_settings.max_concurrent_streams,
self.conn.remote_settings.max_concurrent_streams
)
def _send_pending_requests(self) -> None:
"""Initiate all pending requests from the deque following FIFO
We make sure that at any time {allowed_max_concurrent_streams}
streams are active.
"""
while (
self._pending_request_stream_pool
and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
and self.h2_connected
):
self.metadata['active_streams'] += 1
stream = self._pending_request_stream_pool.popleft()
stream.initiate_request()
self._write_to_transport()
def pop_stream(self, stream_id: int) -> Stream:
"""Perform cleanup when a stream is closed
"""
stream = self.streams.pop(stream_id)
self.metadata['active_streams'] -= 1
self._send_pending_requests()
return stream
def _new_stream(self, request: Request, spider: Spider) -> Stream:
"""Instantiates a new Stream object
"""
stream = Stream(
stream_id=next(self._stream_id_generator),
request=request,
protocol=self,
download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
)
self.streams[stream.stream_id] = stream
return stream
def _write_to_transport(self) -> None:
""" Write data to the underlying transport connection
from the HTTP2 connection instance if any
"""
# Reset the idle timeout as connection is still actively sending data
self.resetTimeout()
data = self.conn.data_to_send()
self.transport.write(data)
def request(self, request: Request, spider: Spider) -> Deferred:
if not isinstance(request, Request):
raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
stream = self._new_stream(request, spider)
d = stream.get_response()
# Add the stream to the request pool
self._pending_request_stream_pool.append(stream)
# If we receive a request when connection is idle
# We need to initiate pending requests
self._send_pending_requests()
return d
def connectionMade(self) -> None:
"""Called by Twisted when the connection is established. We can start
sending some data now: we should open with the connection preamble.
"""
# Initialize the timeout
self.setTimeout(self.IDLE_TIMEOUT)
destination = self.transport.getPeer()
self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
# Initiate H2 Connection
self.conn.initiate_connection()
self._write_to_transport()
def _lose_connection_with_error(self, errors: List[BaseException]) -> None:
"""Helper function to lose the connection with the error sent as a
reason"""
self._conn_lost_errors += errors
self.transport.loseConnection()
def handshakeCompleted(self) -> None:
"""
Close the connection if it's not made via the expected protocol
"""
if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
# we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
def _check_received_data(self, data: bytes) -> None:
"""Checks for edge cases where the connection to remote fails
without raising an appropriate H2Error
Arguments:
data -- Data received from the remote
"""
if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
raise MethodNotAllowed405(self.metadata['ip_address'])
def dataReceived(self, data: bytes) -> None:
# Reset the idle timeout as connection is still actively receiving data
self.resetTimeout()
try:
self._check_received_data(data)
events = self.conn.receive_data(data)
self._handle_events(events)
except H2Error as e:
if isinstance(e, FrameTooLargeError):
# hyper-h2 does not drop the connection in this scenario, we
# need to abort the connection manually.
self._conn_lost_errors += [e]
self.transport.abortConnection()
return
# Save this error as ultimately the connection will be dropped
# internally by hyper-h2. Saved error will be passed to all the streams
# closed with the connection.
self._lose_connection_with_error([e])
finally:
self._write_to_transport()
def timeoutConnection(self) -> None:
"""Called when the connection times out.
We lose the connection with TimeoutError"""
# Check whether there are open streams. If there are, we're going to
# want to use the error code PROTOCOL_ERROR. If there aren't, use
# NO_ERROR.
if (
self.conn.open_outbound_streams > 0
or self.conn.open_inbound_streams > 0
or self.metadata['active_streams'] > 0
):
error_code = ErrorCodes.PROTOCOL_ERROR
else:
error_code = ErrorCodes.NO_ERROR
self.conn.close_connection(error_code=error_code)
self._write_to_transport()
self._lose_connection_with_error([
TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
])
def connectionLost(self, reason: Failure = connectionDone) -> None:
"""Called by Twisted when the transport connection is lost.
No need to write anything to transport here.
"""
# Cancel the timeout if not done yet
self.setTimeout(None)
# Notify the connection pool instance such that no new requests are
# sent over current connection
if not reason.check(connectionDone):
self._conn_lost_errors.append(reason)
self._conn_lost_deferred.callback(self._conn_lost_errors)
for stream in self.streams.values():
if stream.metadata['request_sent']:
close_reason = StreamCloseReason.CONNECTION_LOST
else:
close_reason = StreamCloseReason.INACTIVE
stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
self.metadata['active_streams'] -= len(self.streams)
self.streams.clear()
self._pending_request_stream_pool.clear()
self.conn.close_connection()
def _handle_events(self, events: List[Event]) -> None:
"""Private method which acts as a bridge between the events
received from the HTTP/2 data and IH2EventsHandler
Arguments:
events -- A list of events that the remote peer triggered by sending data
"""
for event in events:
if isinstance(event, ConnectionTerminated):
self.connection_terminated(event)
elif isinstance(event, DataReceived):
self.data_received(event)
elif isinstance(event, ResponseReceived):
self.response_received(event)
elif isinstance(event, StreamEnded):
self.stream_ended(event)
elif isinstance(event, StreamReset):
self.stream_reset(event)
elif isinstance(event, WindowUpdated):
self.window_updated(event)
elif isinstance(event, SettingsAcknowledged):
self.settings_acknowledged(event)
elif isinstance(event, UnknownFrameReceived):
logger.warning('Unknown frame received: %s', event.frame)
# Event handler functions starts here
def connection_terminated(self, event: ConnectionTerminated) -> None:
self._lose_connection_with_error([
RemoteTerminatedConnection(self.metadata['ip_address'], event)
])
def data_received(self, event: DataReceived) -> None:
try:
stream = self.streams[event.stream_id]
except KeyError:
pass # We ignore server-initiated events
else:
stream.receive_data(event.data, event.flow_controlled_length)
def response_received(self, event: ResponseReceived) -> None:
try:
stream = self.streams[event.stream_id]
except KeyError:
pass # We ignore server-initiated events
else:
stream.receive_headers(event.headers)
def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
self.metadata['settings_acknowledged'] = True
# Send off all the pending requests as now we have
# established a proper HTTP/2 connection
self._send_pending_requests()
# Update certificate when our HTTP/2 connection is established
self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
def stream_ended(self, event: StreamEnded) -> None:
try:
stream = self.pop_stream(event.stream_id)
except KeyError:
pass # We ignore server-initiated events
else:
stream.close(StreamCloseReason.ENDED, from_protocol=True)
def stream_reset(self, event: StreamReset) -> None:
try:
stream = self.pop_stream(event.stream_id)
except KeyError:
pass # We ignore server-initiated events
else:
stream.close(StreamCloseReason.RESET, from_protocol=True)
def window_updated(self, event: WindowUpdated) -> None:
if event.stream_id != 0:
self.streams[event.stream_id].receive_window_update()
else:
# Send leftover data for all the streams
for stream in self.streams.values():
stream.receive_window_update()
@implementer(IProtocolNegotiationFactory)
class H2ClientFactory(Factory):
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
self.uri = uri
self.settings = settings
self.conn_lost_deferred = conn_lost_deferred
def buildProtocol(self, addr) -> H2ClientProtocol:
return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
def acceptableProtocols(self) -> List[bytes]:
return [PROTOCOL_NAME]

470
scrapy/core/http2/stream.py Normal file
View File

@ -0,0 +1,470 @@
import logging
from enum import Enum
from io import BytesIO
from urllib.parse import urlparse
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
from hpack import HeaderTuple
from twisted.internet.defer import Deferred, CancelledError
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed
from scrapy.http import Request
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
if TYPE_CHECKING:
from scrapy.core.http2.protocol import H2ClientProtocol
logger = logging.getLogger(__name__)
class InactiveStreamClosed(ConnectionClosed):
"""Connection was closed without sending request headers
of the stream. This happens when a stream is waiting for other
streams to close and connection is lost."""
def __init__(self, request: Request) -> None:
self.request = request
def __str__(self) -> str:
return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
class InvalidHostname(H2Error):
def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
self.request = request
self.expected_hostname = expected_hostname
self.expected_netloc = expected_netloc
def __str__(self) -> str:
return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
class StreamCloseReason(Enum):
# Received a StreamEnded event from the remote
ENDED = 1
# Received a StreamReset event -- ended abruptly
RESET = 2
# Transport connection was lost
CONNECTION_LOST = 3
# Expected response body size is more than allowed limit
MAXSIZE_EXCEEDED = 4
# Response deferred is cancelled by the client
# (happens when client called response_deferred.cancel())
CANCELLED = 5
# Connection lost and the stream was not initiated
INACTIVE = 6
# The hostname of the request is not same as of connected peer hostname
# As a result sending this request will the end the connection
INVALID_HOSTNAME = 7
class Stream:
"""Represents a single HTTP/2 Stream.
Stream is a bidirectional flow of bytes within an established connection,
which may carry one or more messages. Handles the transfer of HTTP Headers
and Data frames.
Role of this class is to
1. Combine all the data frames
"""
def __init__(
self,
stream_id: int,
request: Request,
protocol: "H2ClientProtocol",
download_maxsize: int = 0,
download_warnsize: int = 0,
) -> None:
"""
Arguments:
stream_id -- Unique identifier for the stream within a single HTTP/2 connection
request -- The HTTP request associated to the stream
protocol -- Parent H2ClientProtocol instance
"""
self.stream_id: int = stream_id
self._request: Request = request
self._protocol: "H2ClientProtocol" = protocol
self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
# Metadata of an HTTP/2 connection stream
# initialized when stream is instantiated
self.metadata: Dict = {
'request_content_length': 0 if self._request.body is None else len(self._request.body),
# Flag to keep track whether the stream has initiated the request
'request_sent': False,
# Flag to track whether we have logged about exceeding download warnsize
'reached_warnsize': False,
# Each time we send a data frame, we will decrease value by the amount send.
'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
# Flag to keep track whether client (self) have closed this stream
'stream_closed_local': False,
# Flag to keep track whether the server has closed the stream
'stream_closed_server': False,
}
# Private variable used to build the response
# this response is then converted to appropriate Response class
# passed to the response deferred callback
self._response: Dict = {
# Data received frame by frame from the server is appended
# and passed to the response Deferred when completely received.
'body': BytesIO(),
# The amount of data received that counts against the
# flow control window
'flow_controlled_size': 0,
# Headers received after sending the request
'headers': Headers({}),
}
def _cancel(_) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
if self.metadata['request_sent']:
self.reset_stream(StreamCloseReason.CANCELLED)
else:
self.close(StreamCloseReason.CANCELLED)
self._deferred_response = Deferred(_cancel)
def __str__(self) -> str:
return f'Stream(id={self.stream_id!r})'
__repr__ = __str__
@property
def _log_warnsize(self) -> bool:
"""Checks if we have received data which exceeds the download warnsize
and whether we have not already logged about it.
Returns:
True if both the above conditions hold true
False if any of the conditions is false
"""
content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
return (
self._download_warnsize
and (
self._response['flow_controlled_size'] > self._download_warnsize
or content_length_header > self._download_warnsize
)
and not self.metadata['reached_warnsize']
)
def get_response(self) -> Deferred:
"""Simply return a Deferred which fires when response
from the asynchronous request is available
"""
return self._deferred_response
def check_request_url(self) -> bool:
# Make sure that we are sending the request to the correct URL
url = urlparse(self._request.url)
return (
url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
)
def _get_request_headers(self) -> List[Tuple[str, str]]:
url = urlparse(self._request.url)
path = url.path
if url.query:
path += '?' + url.query
# This pseudo-header field MUST NOT be empty for "http" or "https"
# URIs; "http" or "https" URIs that do not contain a path component
# MUST include a value of '/'. The exception to this rule is an
# OPTIONS request for an "http" or "https" URI that does not include
# a path component; these MUST include a ":path" pseudo-header field
# with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
if not path:
path = '*' if self._request.method == 'OPTIONS' else '/'
# Make sure pseudo-headers comes before all the other headers
headers = [
(':method', self._request.method),
(':authority', url.netloc),
]
# The ":scheme" and ":path" pseudo-header fields MUST
# be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
if self._request.method != 'CONNECT':
headers += [
(':scheme', self._protocol.metadata['uri'].scheme),
(':path', path),
]
content_length = str(len(self._request.body))
headers.append(('Content-Length', content_length))
content_length_name = self._request.headers.normkey(b'Content-Length')
for name, values in self._request.headers.items():
for value in values:
value = str(value, 'utf-8')
if name == content_length_name:
if value != content_length:
logger.warning(
'Ignoring bad Content-Length header %r of request %r, '
'sending %r instead',
value,
self._request,
content_length,
)
continue
headers.append((str(name, 'utf-8'), value))
return headers
def initiate_request(self) -> None:
if self.check_request_url():
headers = self._get_request_headers()
self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
self.metadata['request_sent'] = True
self.send_data()
else:
# Close this stream calling the response errback
# Note that we have not sent any headers
self.close(StreamCloseReason.INVALID_HOSTNAME)
def send_data(self) -> None:
"""Called immediately after the headers are sent. Here we send all the
data as part of the request.
If the content length is 0 initially then we end the stream immediately and
wait for response data.
Warning: Only call this method when stream not closed from client side
and has initiated request already by sending HEADER frame. If not then
stream will raise ProtocolError (raise by h2 state machine).
"""
if self.metadata['stream_closed_local']:
raise StreamClosedError(self.stream_id)
# Firstly, check what the flow control window is for current stream.
window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
# Next, check what the maximum frame size is.
max_frame_size = self._protocol.conn.max_outbound_frame_size
# We will send no more than the window size or the remaining file size
# of data in this call, whichever is smaller.
bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
# We now need to send a number of data frames.
while bytes_to_send_size > 0:
chunk_size = min(bytes_to_send_size, max_frame_size)
data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
bytes_to_send_size = bytes_to_send_size - chunk_size
self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
# End the stream if no more data needs to be send
if self.metadata['remaining_content_length'] == 0:
self._protocol.conn.end_stream(self.stream_id)
# Q. What about the rest of the data?
# Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
def receive_window_update(self) -> None:
"""Flow control window size was changed.
Send data that earlier could not be sent as we were
blocked behind the flow control.
"""
if (
self.metadata['remaining_content_length']
and not self.metadata['stream_closed_server']
and self.metadata['request_sent']
):
self.send_data()
def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
self._response['body'].write(data)
self._response['flow_controlled_size'] += flow_controlled_length
# We check maxsize here in case the Content-Length header was not received
if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
return
if self._log_warnsize:
self.metadata['reached_warnsize'] = True
warning_msg = (
f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
f'warn size ({self._download_warnsize}) in request {self._request}'
)
logger.warning(warning_msg)
# Acknowledge the data received
self._protocol.conn.acknowledge_received_data(
self._response['flow_controlled_size'],
self.stream_id
)
def receive_headers(self, headers: List[HeaderTuple]) -> None:
for name, value in headers:
self._response['headers'][name] = value
# Check if we exceed the allowed max data size which can be received
expected_size = int(self._response['headers'].get(b'Content-Length', -1))
if self._download_maxsize and expected_size > self._download_maxsize:
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
return
if self._log_warnsize:
self.metadata['reached_warnsize'] = True
warning_msg = (
f'Expected response size ({expected_size}) larger than '
f'download warn size ({self._download_warnsize}) in request {self._request}'
)
logger.warning(warning_msg)
def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
"""Close this stream by sending a RST_FRAME to the remote peer"""
if self.metadata['stream_closed_local']:
raise StreamClosedError(self.stream_id)
# Clear buffer earlier to avoid keeping data in memory for a long time
self._response['body'].truncate(0)
self.metadata['stream_closed_local'] = True
self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
self.close(reason)
def close(
self,
reason: StreamCloseReason,
errors: Optional[List[BaseException]] = None,
from_protocol: bool = False,
) -> None:
"""Based on the reason sent we will handle each case.
"""
if self.metadata['stream_closed_server']:
raise StreamClosedError(self.stream_id)
if not isinstance(reason, StreamCloseReason):
raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
# Have default value of errors as an empty list as
# some cases can add a list of exceptions
errors = errors or []
if not from_protocol:
self._protocol.pop_stream(self.stream_id)
self.metadata['stream_closed_server'] = True
# We do not check for Content-Length or Transfer-Encoding in response headers
# and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
# a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6)
# NOTE: Order of handling the events is important here
# As we immediately cancel the request when maxsize is exceeded while
# receiving DATA_FRAME's when we have received the headers (not
# having Content-Length)
if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
expected_size = int(self._response['headers'].get(
b'Content-Length',
self._response['flow_controlled_size'])
)
error_msg = (
f'Cancelling download of {self._request.url}: received response '
f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
)
logger.error(error_msg)
self._deferred_response.errback(CancelledError(error_msg))
elif reason is StreamCloseReason.ENDED:
self._fire_response_deferred()
# Stream was abruptly ended here
elif reason is StreamCloseReason.CANCELLED:
# Client has cancelled the request. Remove all the data
# received and fire the response deferred with no flags set
# NOTE: The data is already flushed in Stream.reset_stream() called
# immediately when the stream needs to be cancelled
# There maybe no :status in headers, we make
# HTTP Status Code: 499 - Client Closed Request
self._response['headers'][':status'] = '499'
self._fire_response_deferred()
elif reason is StreamCloseReason.RESET:
self._deferred_response.errback(ResponseFailed([
Failure(
f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
ProtocolError
)
]))
elif reason is StreamCloseReason.CONNECTION_LOST:
self._deferred_response.errback(ResponseFailed(errors))
elif reason is StreamCloseReason.INACTIVE:
errors.insert(0, InactiveStreamClosed(self._request))
self._deferred_response.errback(ResponseFailed(errors))
else:
assert reason is StreamCloseReason.INVALID_HOSTNAME
self._deferred_response.errback(InvalidHostname(
self._request,
str(self._protocol.metadata['uri'].host, 'utf-8'),
f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
))
def _fire_response_deferred(self) -> None:
"""Builds response from the self._response dict
and fires the response deferred callback with the
generated response instance"""
body = self._response['body'].getvalue()
response_cls = responsetypes.from_args(
headers=self._response['headers'],
url=self._request.url,
body=body,
)
response = response_cls(
url=self._request.url,
status=int(self._response['headers'][':status']),
headers=self._response['headers'],
body=body,
request=self._request,
certificate=self._protocol.metadata['certificate'],
ip_address=self._protocol.metadata['ip_address'],
protocol='h2',
)
self._deferred_response.callback(response)

308
scrapy/core/scheduler.py
View File

@ -1,46 +1,179 @@
import os
import json
import logging
import warnings
from os.path import join, exists
import os
from abc import abstractmethod
from os.path import exists, join
from typing import Optional, Type, TypeVar
from queuelib import PriorityQueue
from twisted.internet.defer import Deferred
from scrapy.utils.misc import load_object, create_instance
from scrapy.crawler import Crawler
from scrapy.http.request import Request
from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.misc import create_instance, load_object
logger = logging.getLogger(__name__)
class Scheduler:
class BaseSchedulerMeta(type):
"""
Scrapy Scheduler. It allows to enqueue requests and then get
a next request to download. Scheduler is also handling duplication
filtering, via dupefilter.
Prioritization and queueing is not performed by the Scheduler.
User sets ``priority`` field for each Request, and a PriorityQueue
(defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
to dequeue requests in a desired order.
Scheduler uses two PriorityQueue instances, configured to work in-memory
and on-disk (optional). When on-disk queue is present, it is used by
default, and an in-memory queue is used as a fallback for cases where
a disk queue can't handle a request (can't serialize it).
:setting:`SCHEDULER_MEMORY_QUEUE` and
:setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
which PriorityQueue instances would be instantiated with, to keep requests
on disk and in memory respectively.
Overall, Scheduler is an object which holds several PriorityQueue instances
(in-memory and on-disk) and implements fallback logic for them.
Also, it handles dupefilters.
Metaclass to check scheduler classes against the necessary interface
"""
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
logunser=False, stats=None, pqclass=None, crawler=None):
def __instancecheck__(cls, instance):
return cls.__subclasscheck__(type(instance))
def __subclasscheck__(cls, subclass):
return (
hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
and hasattr(subclass, "next_request") and callable(subclass.next_request)
)
class BaseScheduler(metaclass=BaseSchedulerMeta):
"""
The scheduler component is responsible for storing requests received from
the engine, and feeding them back upon request (also to the engine).
The original sources of said requests are:
* Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks
* Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods
* Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods
The order in which the scheduler returns its stored requests (via the ``next_request`` method)
plays a great part in determining the order in which those requests are downloaded.
The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
"""
@classmethod
def from_crawler(cls, crawler: Crawler):
"""
Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
"""
return cls()
def open(self, spider: Spider) -> Optional[Deferred]:
"""
Called when the spider is opened by the engine. It receives the spider
instance as argument and it's useful to execute initialization code.
:param spider: the spider object for the current crawl
:type spider: :class:`~scrapy.spiders.Spider`
"""
pass
def close(self, reason: str) -> Optional[Deferred]:
"""
Called when the spider is closed by the engine. It receives the reason why the crawl
finished as argument and it's useful to execute cleaning code.
:param reason: a string which describes the reason why the spider was closed
:type reason: :class:`str`
"""
pass
@abstractmethod
def has_pending_requests(self) -> bool:
"""
``True`` if the scheduler has enqueued requests, ``False`` otherwise
"""
raise NotImplementedError()
@abstractmethod
def enqueue_request(self, request: Request) -> bool:
"""
Process a request received by the engine.
Return ``True`` if the request is stored correctly, ``False`` otherwise.
If ``False``, the engine will fire a ``request_dropped`` signal, and
will not make further attempts to schedule the request at a later time.
For reference, the default Scrapy scheduler returns ``False`` when the
request is rejected by the dupefilter.
"""
raise NotImplementedError()
@abstractmethod
def next_request(self) -> Optional[Request]:
"""
Return the next :class:`~scrapy.http.Request` to be processed, or ``None``
to indicate that there are no requests to be considered ready at the moment.
Returning ``None`` implies that no request from the scheduler will be sent
to the downloader in the current reactor cycle. The engine will continue
calling ``next_request`` until ``has_pending_requests`` is ``False``.
"""
raise NotImplementedError()
SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
class Scheduler(BaseScheduler):
"""
Default Scrapy scheduler. This implementation also handles duplication
filtering via the :setting:`dupefilter <DUPEFILTER_CLASS>`.
This scheduler stores requests into several priority queues (defined by the
:setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues
are backed by either memory or disk based queues (respectively defined by the
:setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings).
Request prioritization is almost entirely delegated to the priority queue. The only
prioritization performed by this scheduler is using the disk-based queue if present
(i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based
queue if a serialization error occurs. If the disk queue is not present, the memory one
is used directly.
:param dupefilter: An object responsible for checking and filtering duplicate requests.
The value for the :setting:`DUPEFILTER_CLASS` setting is used by default.
:type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar:
any class that implements the `BaseDupeFilter` interface
:param jobdir: The path of a directory to be used for persisting the crawl's state.
The value for the :setting:`JOBDIR` setting is used by default.
See :ref:`topics-jobs`.
:type jobdir: :class:`str` or ``None``
:param dqclass: A class to be used as persistent request queue.
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
:type dqclass: class
:param mqclass: A class to be used as non-persistent request queue.
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
:type mqclass: class
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
:type logunser: bool
:param stats: A stats collector object to record stats about the request scheduling process.
The value for the :setting:`STATS_CLASS` setting is used by default.
:type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar:
any class that implements the `StatsCollector` interface
:param pqclass: A class to be used as priority queue for requests.
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
:type pqclass: class
:param crawler: The crawler object corresponding to the current crawl.
:type crawler: :class:`scrapy.crawler.Crawler`
"""
def __init__(
self,
dupefilter,
jobdir: Optional[str] = None,
dqclass=None,
mqclass=None,
logunser: bool = False,
stats=None,
pqclass=None,
crawler: Optional[Crawler] = None,
):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.pqclass = pqclass
@ -51,42 +184,57 @@ class Scheduler:
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
dupefilter = create_instance(dupefilter_cls, settings, crawler)
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
if pqclass is PriorityQueue:
warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
" is no longer supported because of API changes; "
"please use 'scrapy.pqueues.ScrapyPriorityQueue'",
ScrapyDeprecationWarning)
from scrapy.pqueues import ScrapyPriorityQueue
pqclass = ScrapyPriorityQueue
def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
"""
Factory method, initializes the scheduler with arguments taken from the crawl settings
"""
dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
return cls(
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
jobdir=job_dir(crawler.settings),
dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
stats=crawler.stats,
pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
crawler=crawler,
)
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('SCHEDULER_DEBUG')
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
mqclass=mqclass, crawler=crawler)
def has_pending_requests(self):
def has_pending_requests(self) -> bool:
return len(self) > 0
def open(self, spider):
def open(self, spider: Spider) -> Optional[Deferred]:
"""
(1) initialize the memory queue
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
(3) return the result of the dupefilter's ``open`` method
"""
self.spider = spider
self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason):
if self.dqs:
def close(self, reason: str) -> Optional[Deferred]:
"""
(1) dump pending requests to disk if there is a disk queue
(2) return the result of the dupefilter's ``close`` method
"""
if self.dqs is not None:
state = self.dqs.close()
assert isinstance(self.dqdir, str)
self._write_dqs_state(self.dqdir, state)
return self.df.close(reason)
def enqueue_request(self, request):
def enqueue_request(self, request: Request) -> bool:
"""
Unless the received request is filtered out by the Dupefilter, attempt to push
it into the disk queue, falling back to pushing it into the memory queue.
Increment the appropriate stats, such as: ``scheduler/enqueued``,
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
Return ``True`` if the request was stored successfully, ``False`` otherwise.
"""
if not request.dont_filter and self.df.request_seen(request):
self.df.log(request, self.spider)
return False
@ -99,24 +247,35 @@ class Scheduler:
self.stats.inc_value('scheduler/enqueued', spider=self.spider)
return True
def next_request(self):
def next_request(self) -> Optional[Request]:
"""
Return a :class:`~scrapy.http.Request` object from the memory queue,
falling back to the disk queue if the memory queue is empty.
Return ``None`` if there are no more enqueued requests.
Increment the appropriate stats, such as: ``scheduler/dequeued``,
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
"""
request = self.mqs.pop()
if request:
if request is not None:
self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
else:
request = self._dqpop()
if request:
if request is not None:
self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
if request:
if request is not None:
self.stats.inc_value('scheduler/dequeued', spider=self.spider)
return request
def __len__(self):
return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs)
def __len__(self) -> int:
"""
Return the total amount of enqueued requests
"""
return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs)
def _dqpush(self, request):
def _dqpush(self, request: Request) -> bool:
if self.dqs is None:
return
return False
try:
self.dqs.push(request)
except ValueError as e: # non serializable request
@ -127,18 +286,18 @@ class Scheduler:
logger.warning(msg, {'request': request, 'reason': e},
exc_info=True, extra={'spider': self.spider})
self.logunser = False
self.stats.inc_value('scheduler/unserializable',
spider=self.spider)
return
self.stats.inc_value('scheduler/unserializable', spider=self.spider)
return False
else:
return True
def _mqpush(self, request):
def _mqpush(self, request: Request) -> None:
self.mqs.push(request)
def _dqpop(self):
if self.dqs:
def _dqpop(self) -> Optional[Request]:
if self.dqs is not None:
return self.dqs.pop()
return None
def _mq(self):
""" Create a new priority queue instance, with in-memory storage """
@ -162,21 +321,22 @@ class Scheduler:
{'queuesize': len(q)}, extra={'spider': self.spider})
return q
def _dqdir(self, jobdir):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
if jobdir:
if jobdir is not None:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
return None
def _read_dqs_state(self, dqdir):
def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json')
if not exists(path):
return ()
return []
with open(path) as f:
return json.load(f)
def _write_dqs_state(self, dqdir, state):
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f:
json.dump(state, f)

Some files were not shown because too many files have changed in this diff Show More