mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names
This commit is contained in:
Adrián Chaves
commit
e8503217fb
|
|
@ -8,6 +8,7 @@ skips:
|
|||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B324
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.4.1
|
||||
current_version = 2.6.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
[flake8]
|
||||
|
||||
max-line-length = 119
|
||||
ignore = W503
|
||||
|
||||
exclude =
|
||||
# Exclude files that are meant to provide top-level imports
|
||||
# E402: Module level import not at top of file
|
||||
# F401: Module imported but unused
|
||||
scrapy/__init__.py E402
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/linkextractors/__init__.py E402 F401
|
||||
scrapy/selector/__init__.py F401
|
||||
scrapy/spiders/__init__.py E402 F401
|
||||
|
||||
# Issues pending a review:
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
name: Checks
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
checks:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: security
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: flake8
|
||||
# Pylint requires installing reppy, which does not support Python 3.9
|
||||
# https://github.com/seomoz/reppy/issues/122
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
name: Run test suite
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test-windows:
|
||||
name: "Windows Tests"
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [windows-latest]
|
||||
python-version: [3.7, 3.8]
|
||||
env: [TOXENV: py]
|
||||
include:
|
||||
- os: windows-latest
|
||||
python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run test suite
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
name: Publish
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-18.04
|
||||
if: startsWith(github.event.ref, 'refs/tags/')
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Check Tag
|
||||
id: check-release-tag
|
||||
run: |
|
||||
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
|
||||
echo ::set-output name=release_tag::true
|
||||
fi
|
||||
|
||||
- name: Publish to PyPI
|
||||
if: steps.check-release-tag.outputs.release_tag == 'true'
|
||||
run: |
|
||||
pip install --upgrade setuptools wheel twine
|
||||
python setup.py sdist bdist_wheel
|
||||
export TWINE_USERNAME=__token__
|
||||
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
|
||||
twine upload dist/*
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
name: macOS
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: macos-10.15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox -e py
|
||||
|
||||
- name: Upload coverage report
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
name: Ubuntu
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-18.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.9-v7.3.9
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: pinned
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.7-v7.3.5
|
||||
|
||||
# extras
|
||||
# extra-deps includes reppy, which does not support Python 3.9
|
||||
# https://github.com/seomoz/reppy/issues/122
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install system libraries
|
||||
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
|
||||
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
- name: Upload coverage report
|
||||
run: bash <(curl -s https://codecov.io/bash)
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
name: Windows
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
@ -14,6 +14,8 @@ htmlcov/
|
|||
.coverage
|
||||
.pytest_cache/
|
||||
.coverage.*
|
||||
coverage.*
|
||||
test-output.*
|
||||
.cache/
|
||||
.mypy_cache/
|
||||
/tests/keys/localhost.crt
|
||||
|
|
|
|||
|
|
@ -3,10 +3,15 @@ formats: all
|
|||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
fail_on_warning: true
|
||||
|
||||
build:
|
||||
os: ubuntu-20.04
|
||||
tools:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
|
||||
python: "3.10" # Keep in sync with .github/workflows/checks.yml
|
||||
|
||||
python:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
|
||||
version: 3.7 # Keep in sync with .travis.yml
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- path: .
|
||||
|
|
|
|||
75
.travis.yml
75
.travis.yml
|
|
@ -1,75 +0,0 @@
|
|||
language: python
|
||||
dist: xenial
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d\.\d+$/
|
||||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=security
|
||||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
- env: TOXENV=typing
|
||||
python: 3.8
|
||||
|
||||
- env: TOXENV=pinned
|
||||
python: 3.6.1
|
||||
- env: TOXENV=asyncio-pinned
|
||||
python: 3.6.1
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
- pip install -U tox twine wheel codecov
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- codecov
|
||||
notifications:
|
||||
irc:
|
||||
use_notice: true
|
||||
skip_join: true
|
||||
channels:
|
||||
- irc.freenode.org#scrapy
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
deploy:
|
||||
provider: pypi
|
||||
distributions: "sdist bdist_wheel"
|
||||
user: scrapy
|
||||
password:
|
||||
secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
|
||||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
4
AUTHORS
4
AUTHORS
|
|
@ -1,8 +1,8 @@
|
|||
Scrapy was brought to life by Shane Evans while hacking a scraping framework
|
||||
prototype for Mydeco (mydeco.com). It soon became maintained, extended and
|
||||
improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
|
||||
bootstrap the project. In mid-2011, Scrapinghub became the new official
|
||||
maintainer.
|
||||
bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
|
||||
official maintainer.
|
||||
|
||||
Here is the list of the primary authors & contributors:
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
|
|||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at opensource@scrapinghub.com. All
|
||||
reported by contacting the project team at opensource@zyte.com. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
|
|
@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version].
|
|||
|
||||
[homepage]: http://contributor-covenant.org
|
||||
[version]: http://contributor-covenant.org/version/1/4/
|
||||
|
||||
For answers to common questions about this code of conduct, see
|
||||
https://www.contributor-covenant.org/faq
|
||||
|
|
|
|||
27
README.rst
27
README.rst
|
|
@ -1,3 +1,5 @@
|
|||
.. image:: https://scrapy.org/img/scrapylogo.png
|
||||
|
||||
======
|
||||
Scrapy
|
||||
======
|
||||
|
|
@ -10,9 +12,17 @@ Scrapy
|
|||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: Supported Python Versions
|
||||
|
||||
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
|
||||
:target: https://travis-ci.org/scrapy/scrapy
|
||||
:alt: Build Status
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
|
||||
:alt: Ubuntu
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
|
||||
:alt: macOS
|
||||
|
||||
.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
|
||||
:alt: Windows
|
||||
|
||||
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
|
|
@ -34,13 +44,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
|
|||
crawl websites and extract structured data from their pages. It can be used for
|
||||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
|
||||
contributors`_.
|
||||
|
||||
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
|
||||
.. _Zyte: https://www.zyte.com/
|
||||
|
||||
Check the Scrapy homepage at https://scrapy.org for more information,
|
||||
including a list of features.
|
||||
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.6+
|
||||
* Python 3.7+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
@ -81,7 +98,7 @@ Please note that this project is released with a Contributor Code of Conduct
|
|||
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
|
||||
|
||||
By participating in this project you agree to abide by its terms.
|
||||
Please report unacceptable behavior to opensource@scrapinghub.com.
|
||||
Please report unacceptable behavior to opensource@zyte.com.
|
||||
|
||||
Companies using Scrapy
|
||||
======================
|
||||
|
|
|
|||
39
conftest.py
39
conftest.py
|
|
@ -1,6 +1,9 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from twisted.web.http import H2_ENABLED
|
||||
|
||||
from scrapy.utils.reactor import install_reactor
|
||||
|
||||
from tests.keys import generate_keys
|
||||
|
||||
|
|
@ -18,10 +21,19 @@ collect_ignore = [
|
|||
*_py_files("tests/CrawlerRunner"),
|
||||
]
|
||||
|
||||
for line in open('tests/ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
with open('tests/ignores.txt') as reader:
|
||||
for line in reader:
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
|
||||
if not H2_ENABLED:
|
||||
collect_ignore.extend(
|
||||
(
|
||||
'scrapy/core/downloader/handlers/http2.py',
|
||||
*_py_files("scrapy/core/http2"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
@ -40,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items):
|
|||
pass
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--reactor",
|
||||
default="default",
|
||||
choices=["default", "asyncio"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope='class')
|
||||
def reactor_pytest(request):
|
||||
if not request.cls:
|
||||
|
|
@ -55,5 +75,16 @@ def only_asyncio(request, reactor_pytest):
|
|||
pytest.skip('This test is only run with --reactor=asyncio')
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def only_not_asyncio(request, reactor_pytest):
|
||||
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
|
||||
pytest.skip('This test is only run without --reactor=asyncio')
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
if config.getoption("--reactor") == "asyncio":
|
||||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
|
||||
|
||||
# Generate localhost certificate files, needed by some tests
|
||||
generate_keys()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ PYTHON = python
|
|||
SPHINXOPTS =
|
||||
PAPER =
|
||||
SOURCES =
|
||||
SHELL = /bin/bash
|
||||
SHELL = /usr/bin/env bash
|
||||
|
||||
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
|
||||
-D latex_elements.papersize=$(PAPER) \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
/* Move lists closer to their introducing paragraph */
|
||||
.rst-content .section ol p, .rst-content .section ul p {
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
.rst-content p + ol, .rst-content p + ul {
|
||||
margin-top: -18px; /* Compensates margin-top: 24px of p */
|
||||
}
|
||||
.rst-content dl p + ol, .rst-content dl p + ul {
|
||||
margin-top: -6px; /* Compensates margin-top: 12px of p */
|
||||
}
|
||||
|
|
@ -1,16 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -3,14 +3,9 @@
|
|||
{% block footer %}
|
||||
{{ super() }}
|
||||
<script type="text/javascript">
|
||||
!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
|
||||
analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
|
||||
analytics.page();
|
||||
}}();
|
||||
|
||||
analytics.ready(function () {
|
||||
ga('require', 'linker');
|
||||
ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
|
||||
ga('linker:autoLink', ['zyte.com']);
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
|
|
|||
13
docs/conf.py
13
docs/conf.py
|
|
@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
|
|||
import sphinx_rtd_theme
|
||||
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
|
||||
# The style sheet to use for HTML and HTML Help pages. A file of that name
|
||||
# must exist either in Sphinx' static/ path, or in one of the custom paths
|
||||
# given in html_static_path.
|
||||
|
|
@ -183,6 +182,10 @@ html_copy_source = True
|
|||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'Scrapydoc'
|
||||
|
||||
html_css_files = [
|
||||
'custom.css',
|
||||
]
|
||||
|
||||
|
||||
# Options for LaTeX output
|
||||
# ------------------------
|
||||
|
|
@ -269,7 +272,6 @@ coverage_ignore_pyobjects = [
|
|||
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
|
||||
|
||||
# Never documented before, and deprecated now.
|
||||
r'^scrapy\.item\.DictItem$',
|
||||
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
|
||||
|
||||
# Implementation detail of LxmlLinkExtractor
|
||||
|
|
@ -283,6 +285,7 @@ coverage_ignore_pyobjects = [
|
|||
intersphinx_mapping = {
|
||||
'attrs': ('https://www.attrs.org/en/stable/', None),
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'cryptography' : ('https://cryptography.io/en/latest/', None),
|
||||
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
|
||||
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
|
|
@ -291,7 +294,9 @@ intersphinx_mapping = {
|
|||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
|
||||
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
|
||||
}
|
||||
intersphinx_disabled_reftypes = []
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
|
|
@ -300,10 +305,14 @@ intersphinx_mapping = {
|
|||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"command": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
"reqmeta": "tooltip",
|
||||
"setting": "tooltip",
|
||||
"signal": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
|||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
try:
|
||||
# >2.0.1
|
||||
from sybil.parsers.codeblock import PythonCodeBlockParser
|
||||
except ImportError:
|
||||
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
|
@ -21,7 +25,7 @@ def setup(namespace):
|
|||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
PythonCodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
|
|
|
|||
|
|
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
the tests with Python 3.7 use::
|
||||
|
||||
tox -e py36
|
||||
tox -e py37
|
||||
|
||||
You can also specify a comma-separated list of environments, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py36,py38 -p auto
|
||||
tox -e py37,py38 -p auto
|
||||
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
|
|
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
|
|||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
tox -e py37 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
|
|
|||
69
docs/faq.rst
69
docs/faq.rst
|
|
@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
|
|||
|
||||
See :ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
|
||||
Scrapy crashes with: ImportError: No module named win32api
|
||||
----------------------------------------------------------
|
||||
|
||||
You need to install `pywin32`_ because of `this Twisted bug`_.
|
||||
|
||||
.. _pywin32: https://sourceforge.net/projects/pywin32/
|
||||
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
|
||||
|
||||
How can I simulate a user login in my spider?
|
||||
---------------------------------------------
|
||||
|
||||
|
|
@ -145,6 +136,41 @@ How can I make Scrapy consume less memory?
|
|||
|
||||
See previous question.
|
||||
|
||||
How can I prevent memory errors due to many allowed domains?
|
||||
------------------------------------------------------------
|
||||
|
||||
If you have a spider with a long list of
|
||||
:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
|
||||
replacing the default
|
||||
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
|
||||
with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
|
||||
less memory. For example:
|
||||
|
||||
- If your domain names are similar enough, use your own regular expression
|
||||
instead joining the strings in
|
||||
:attr:`~scrapy.Spider.allowed_domains` into a complex regular
|
||||
expression.
|
||||
|
||||
- If you can `meet the installation requirements`_, use pyre2_ instead of
|
||||
Python’s re_ to compile your URL-filtering regular expression. See
|
||||
:issue:`1908`.
|
||||
|
||||
See also other suggestions at `StackOverflow`_.
|
||||
|
||||
.. note:: Remember to disable
|
||||
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
|
||||
your custom implementation::
|
||||
|
||||
SPIDER_MIDDLEWARES = {
|
||||
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
|
||||
'myproject.middlewares.CustomOffsiteMiddleware': 500,
|
||||
}
|
||||
|
||||
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
|
||||
.. _pyre2: https://github.com/andreasvc/pyre2
|
||||
.. _re: https://docs.python.org/library/re.html
|
||||
.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
|
||||
|
||||
Can I use Basic HTTP Authentication in my spiders?
|
||||
--------------------------------------------------
|
||||
|
||||
|
|
@ -315,6 +341,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items
|
|||
|
||||
You may need to remove namespaces. See :ref:`removing-namespaces`.
|
||||
|
||||
|
||||
.. _faq-split-item:
|
||||
|
||||
How to split an item into multiple items in an item pipeline?
|
||||
|
|
@ -363,15 +390,27 @@ How can I cancel the download of a given response?
|
|||
--------------------------------------------------
|
||||
|
||||
In some situations, it might be useful to stop the download of a certain response.
|
||||
For instance, if you only need the first part of a large response and you would like
|
||||
to save resources by avoiding the download of the whole body.
|
||||
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
|
||||
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
|
||||
the :ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
For instance, sometimes you can determine whether or not you need the full contents
|
||||
of a response by inspecting its headers or the first bytes of its body. In that case,
|
||||
you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
|
||||
or :class:`~scrapy.signals.headers_received` signals and raising a
|
||||
:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
|
||||
:ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
|
||||
|
||||
Running ``runspider`` I get ``error: No spider found in file: <filename>``
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
This may happen if your Scrapy project has a spider module with a name that
|
||||
conflicts with the name of one of the `Python standard library modules`_, such
|
||||
as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
|
||||
See :issue:`2680`.
|
||||
|
||||
|
||||
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
|
||||
.. _Python standard library modules: https://docs.python.org/py-modindex.html
|
||||
.. _Python package: https://pypi.org/
|
||||
.. _user agents: https://en.wikipedia.org/wiki/User_agent
|
||||
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
|
||||
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
|
||||
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
|
|
@ -12,6 +12,8 @@ testing.
|
|||
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
|
||||
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
|
||||
|
||||
.. _getting-help:
|
||||
|
||||
Getting help
|
||||
============
|
||||
|
||||
|
|
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
|
|||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
|
||||
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
|
||||
|
||||
|
||||
First steps
|
||||
|
|
@ -227,6 +231,7 @@ Extending Scrapy
|
|||
topics/extensions
|
||||
topics/api
|
||||
topics/signals
|
||||
topics/scheduler
|
||||
topics/exporters
|
||||
|
||||
|
||||
|
|
@ -248,6 +253,9 @@ Extending Scrapy
|
|||
:doc:`topics/signals`
|
||||
See all available signals and how to work with them.
|
||||
|
||||
:doc:`topics/scheduler`
|
||||
Understand the scheduler component.
|
||||
|
||||
:doc:`topics/exporters`
|
||||
Quickly export your scraped items to a file (XML, CSV, etc).
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Examples
|
|||
The best way to learn is with examples, and Scrapy is no exception. For this
|
||||
reason, there is an example Scrapy project named quotesbot_, that you can use to
|
||||
play and learn more about Scrapy. It contains two spiders for
|
||||
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
expressions.
|
||||
|
||||
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ Installation guide
|
|||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.6+, either the CPython implementation (default) or
|
||||
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
|
||||
Scrapy requires Python 3.7+, either the CPython implementation (default) or
|
||||
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
.. _intro-install-scrapy:
|
||||
|
||||
Installing Scrapy
|
||||
=================
|
||||
|
|
@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
|
|||
|
||||
pip install Scrapy
|
||||
|
||||
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
|
||||
to avoid conflicting with your system packages.
|
||||
|
||||
Note that sometimes this may require solving compilation issues for some Scrapy
|
||||
dependencies depending on your operating system, so be sure to check the
|
||||
:ref:`intro-install-platform-notes`.
|
||||
|
||||
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
|
||||
to avoid conflicting with your system packages.
|
||||
|
||||
For more detailed and platform specifics instructions, as well as
|
||||
troubleshooting information, read on.
|
||||
|
||||
|
|
@ -51,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
|
|||
* `twisted`_, an asynchronous networking framework
|
||||
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
|
||||
|
||||
The minimal versions which Scrapy is tested against are:
|
||||
|
||||
* Twisted 14.0
|
||||
* lxml 3.4
|
||||
* pyOpenSSL 0.14
|
||||
|
||||
Scrapy may work with older versions of these packages
|
||||
but it is not guaranteed it will continue working
|
||||
because it’s not being tested against them.
|
||||
|
||||
Some of these packages themselves depends on non-Python packages
|
||||
that might require additional installation steps depending on your platform.
|
||||
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
|
||||
|
|
@ -69,10 +60,9 @@ In case of any trouble related to these dependencies,
|
|||
please refer to their respective installation instructions:
|
||||
|
||||
* `lxml installation`_
|
||||
* `cryptography installation`_
|
||||
* :doc:`cryptography installation <cryptography:installation>`
|
||||
|
||||
.. _lxml installation: https://lxml.de/installation.html
|
||||
.. _cryptography installation: https://cryptography.io/en/latest/installation/
|
||||
|
||||
|
||||
.. _intro-using-virtualenv:
|
||||
|
|
@ -118,6 +108,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
|
|||
|
||||
conda install -c conda-forge scrapy
|
||||
|
||||
To install Scrapy on Windows using ``pip``:
|
||||
|
||||
.. warning::
|
||||
This installation method requires “Microsoft Visual C++” for installing some
|
||||
Scrapy dependencies, which demands significantly more disk space than Anaconda.
|
||||
|
||||
#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
|
||||
|
||||
#. Run the Visual Studio Installer.
|
||||
|
||||
#. Under the Workloads section, select **C++ build tools**.
|
||||
|
||||
#. Check the installation details and make sure following packages are selected as optional components:
|
||||
|
||||
* **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
|
||||
|
||||
* **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0))
|
||||
|
||||
#. Install the Visual Studio Build Tools.
|
||||
|
||||
Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
|
||||
|
||||
.. _intro-install-ubuntu:
|
||||
|
||||
|
|
@ -169,7 +180,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
|
|||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
* *(Recommended)* **Don't** use system Python. Install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
|
|
@ -213,8 +224,8 @@ For PyPy3, only Linux installation was tested.
|
|||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependencies will be built during installation.
|
||||
On macOS, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
On macOS, you are likely to face an issue with building the Cryptography
|
||||
dependency. The solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing Scrapy). Installing on Linux has no special
|
||||
|
|
@ -265,10 +276,10 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
|||
.. _cryptography: https://cryptography.io/en/latest/
|
||||
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
|
||||
.. _setuptools: https://pypi.python.org/pypi/setuptools
|
||||
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
|
||||
.. _homebrew: https://brew.sh/
|
||||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
.. _Anaconda: https://docs.anaconda.com/anaconda/
|
||||
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
|
||||
.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
|
||||
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
Scrapy at a glance
|
||||
==================
|
||||
|
||||
Scrapy is an application framework for crawling web sites and extracting
|
||||
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
|
||||
structured data which can be used for a wide range of useful applications, like
|
||||
data mining, information processing or historical archival.
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
|
|||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
https://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
'https://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Scrapy Tutorial
|
|||
In this tutorial, we'll assume that Scrapy is already installed on your system.
|
||||
If that's not the case, see :ref:`intro-install`.
|
||||
|
||||
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
|
||||
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
|
||||
that lists quotes from famous authors.
|
||||
|
||||
This tutorial will walk you through these tasks:
|
||||
|
|
@ -78,7 +78,7 @@ Our first Spider
|
|||
|
||||
Spiders are classes that you define and that Scrapy uses to scrape information
|
||||
from a website (or a group of websites). They must subclass
|
||||
:class:`~scrapy.spiders.Spider` and define the initial requests to make,
|
||||
:class:`~scrapy.Spider` and define the initial requests to make,
|
||||
optionally how to follow links in the pages, and how to parse the downloaded
|
||||
page content to extract data.
|
||||
|
||||
|
|
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
|
|||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
|
@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named
|
|||
self.log(f'Saved file {filename}')
|
||||
|
||||
|
||||
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
|
||||
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
|
||||
and defines some attributes and methods:
|
||||
|
||||
* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
|
||||
* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be
|
||||
unique within a project, that is, you can't set the same name for different
|
||||
Spiders.
|
||||
|
||||
* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of
|
||||
* :meth:`~scrapy.Spider.start_requests`: must return an iterable of
|
||||
Requests (you can return a list of requests or write a generator function)
|
||||
which the Spider will begin to crawl from. Subsequent requests will be
|
||||
generated successively from these initial requests.
|
||||
|
||||
* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
|
||||
* :meth:`~scrapy.Spider.parse`: a method that will be called to handle
|
||||
the response downloaded for each of the requests made. The response parameter
|
||||
is an instance of :class:`~scrapy.http.TextResponse` that holds
|
||||
the page content and has further helpful methods to handle it.
|
||||
|
||||
The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
|
||||
The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting
|
||||
the scraped data as dicts and also finding new URLs to
|
||||
follow and creating new requests (:class:`~scrapy.http.Request`) from them.
|
||||
follow and creating new requests (:class:`~scrapy.Request`) from them.
|
||||
|
||||
How to run our spider
|
||||
---------------------
|
||||
|
|
@ -143,9 +143,9 @@ similar to this::
|
|||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
|
@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs.
|
|||
What just happened under the hood?
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
|
||||
Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
|
||||
returned by the ``start_requests`` method of the Spider. Upon receiving a
|
||||
response for each one, it instantiates :class:`~scrapy.http.Response` objects
|
||||
and calls the callback method associated with the request (in this case, the
|
||||
|
|
@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the
|
|||
|
||||
A shortcut to the start_requests method
|
||||
---------------------------------------
|
||||
Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method
|
||||
that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs,
|
||||
you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute
|
||||
Instead of implementing a :meth:`~scrapy.Spider.start_requests` method
|
||||
that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
|
||||
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
|
||||
with a list of URLs. This list will then be used by the default implementation
|
||||
of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests
|
||||
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
|
||||
for your spider::
|
||||
|
||||
import scrapy
|
||||
|
|
@ -184,8 +184,8 @@ for your spider::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -194,9 +194,9 @@ for your spider::
|
|||
with open(filename, 'wb') as f:
|
||||
f.write(response.body)
|
||||
|
||||
The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
|
||||
The :meth:`~scrapy.Spider.parse` method will be called to handle each
|
||||
of the requests for those URLs, even though we haven't explicitly told Scrapy
|
||||
to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
|
||||
to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's
|
||||
default callback method, which is called for requests without an explicitly
|
||||
assigned callback.
|
||||
|
||||
|
|
@ -207,7 +207,7 @@ Extracting data
|
|||
The best way to learn how to extract data with Scrapy is trying selectors
|
||||
using the :ref:`Scrapy shell <topics-shell>`. Run::
|
||||
|
||||
scrapy shell 'http://quotes.toscrape.com/page/1/'
|
||||
scrapy shell 'https://quotes.toscrape.com/page/1/'
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "http://quotes.toscrape.com/page/1/"
|
||||
scrapy shell "https://quotes.toscrape.com/page/1/"
|
||||
|
||||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 http://quotes.toscrape.com/page/1/>
|
||||
[s] request <GET https://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 https://quotes.toscrape.com/page/1/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
|
||||
[s] Useful shortcuts:
|
||||
|
|
@ -241,14 +241,14 @@ object:
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
||||
The result of running ``response.css('title')`` is a list-like object called
|
||||
:class:`~scrapy.selector.SelectorList`, which represents a list of
|
||||
:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements
|
||||
:class:`~scrapy.Selector` objects that wrap around XML/HTML elements
|
||||
and allow you to run further queries to fine-grain the selection or extract the
|
||||
data.
|
||||
|
||||
|
|
@ -277,9 +277,19 @@ As an alternative, you could've written:
|
|||
>>> response.css('title::text')[0].get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
|
||||
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
|
||||
find any element matching the selection.
|
||||
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
|
||||
raise an :exc:`IndexError` exception if there are no results::
|
||||
|
||||
>>> response.css('noelement')[0].get()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
IndexError: list index out of range
|
||||
|
||||
You might want to use ``.get()`` directly on the
|
||||
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
|
||||
if there are no results::
|
||||
|
||||
>>> response.css("noelement").get()
|
||||
|
||||
There's a lesson here: for most scraping code, you want it to be resilient to
|
||||
errors due to things not being found on a page, so that even if some parts fail
|
||||
|
|
@ -345,7 +355,7 @@ Extracting quotes and authors
|
|||
Now that you know a bit about selection and extraction, let's complete our
|
||||
spider by writing the code to extract the quotes from the web page.
|
||||
|
||||
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
|
||||
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
|
||||
like this:
|
||||
|
||||
.. code-block:: html
|
||||
|
|
@ -369,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'http://quotes.toscrape.com'
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
@ -434,8 +444,8 @@ in the callback, as you can see below::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -448,9 +458,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
@ -464,7 +474,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports
|
|||
|
||||
scrapy crawl quotes -O quotes.json
|
||||
|
||||
That will generate an ``quotes.json`` file containing all scraped items,
|
||||
That will generate a ``quotes.json`` file containing all scraped items,
|
||||
serialized in `JSON`_.
|
||||
|
||||
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
|
||||
|
|
@ -478,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily
|
|||
append new records to it. It doesn't have the same problem of JSON when you run
|
||||
twice. Also, as each record is a separate line, you can process big files
|
||||
without having to fit everything in memory, there are tools like `JQ`_ to help
|
||||
doing that at the command-line.
|
||||
do that at the command-line.
|
||||
|
||||
In small projects (like the one in this tutorial), that should be enough.
|
||||
However, if you want to perform more complex things with the scraped items, you
|
||||
|
|
@ -495,7 +505,7 @@ Following links
|
|||
===============
|
||||
|
||||
Let's say, instead of just scraping the stuff from the first two pages
|
||||
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
|
||||
Now that you know how to extract data from pages, let's see how to follow links
|
||||
from them.
|
||||
|
|
@ -539,7 +549,7 @@ page, extracting data from it::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -644,7 +654,7 @@ this time for scraping author information::
|
|||
class AuthorSpider(scrapy.Spider):
|
||||
name = 'author'
|
||||
|
||||
start_urls = ['http://quotes.toscrape.com/']
|
||||
start_urls = ['https://quotes.toscrape.com/']
|
||||
|
||||
def parse(self, response):
|
||||
author_page_links = response.css('.author + a')
|
||||
|
|
@ -670,7 +680,7 @@ the pagination links with the ``parse`` callback as we saw before.
|
|||
Here we're passing callbacks to
|
||||
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
|
||||
arguments to make the code shorter; it also works for
|
||||
:class:`~scrapy.http.Request`.
|
||||
:class:`~scrapy.Request`.
|
||||
|
||||
The ``parse_author`` callback defines a helper function to extract and cleanup the
|
||||
data from a CSS query and yields the Python dict with the author data.
|
||||
|
|
@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument::
|
|||
name = "quotes"
|
||||
|
||||
def start_requests(self):
|
||||
url = 'http://quotes.toscrape.com/'
|
||||
url = 'https://quotes.toscrape.com/'
|
||||
tag = getattr(self, 'tag', None)
|
||||
if tag is not None:
|
||||
url = url + 'tag/' + tag
|
||||
|
|
@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument::
|
|||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
will only visit URLs from the ``humor`` tag, such as
|
||||
``http://quotes.toscrape.com/tag/humor``.
|
||||
``https://quotes.toscrape.com/tag/humor``.
|
||||
|
||||
You can :ref:`learn more about handling spider arguments here <spiderargs>`.
|
||||
|
||||
|
|
|
|||
740
docs/news.rst
740
docs/news.rst
|
|
@ -3,6 +3,634 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.6.1:
|
||||
|
||||
Scrapy 2.6.1 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Fixes a regression introduced in 2.6.0 that would unset the request method when
|
||||
following redirects.
|
||||
|
||||
|
||||
.. _release-2.6.0:
|
||||
|
||||
Scrapy 2.6.0 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Security fixes for cookie handling <2.6-security-fixes>`
|
||||
|
||||
* Python 3.10 support
|
||||
|
||||
* :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental, and works out-of-the-box on Windows regardless of your Python
|
||||
version
|
||||
|
||||
* Feed exports now support :class:`pathlib.Path` output paths and per-feed
|
||||
:ref:`item filtering <item-filter>` and
|
||||
:ref:`post-processing <post-processing>`
|
||||
|
||||
.. _2.6-security-fixes:
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies from a
|
||||
controlled domain into your cookiejar that could be sent to other domains
|
||||
not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
|
||||
advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The h2_ dependency is now optional, only needed to
|
||||
:ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
|
||||
|
||||
.. _h2: https://pypi.org/project/h2/
|
||||
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
|
||||
for a non-POST request, now overrides the URL query string, instead of
|
||||
being appended to it. (:issue:`2919`, :issue:`3579`)
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
|
||||
the return value of that function, and not the ``params`` input parameter,
|
||||
will determine the feed URI parameters, unless that return value is
|
||||
``None``. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- In :class:`scrapy.core.engine.ExecutionEngine`, methods
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
|
||||
and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
now raise :exc:`RuntimeError` if called before
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
|
||||
|
||||
These methods used to assume that
|
||||
:attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
|
||||
been defined by a prior call to
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
|
||||
raising :exc:`AttributeError` instead.
|
||||
|
||||
- If the API of the configured :ref:`scheduler <topics-scheduler>` does not
|
||||
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
|
||||
other exceptions would be raised at run time. (:issue:`3559`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
|
||||
now been removed. (:issue:`5393`)
|
||||
|
||||
- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
|
||||
been removed. (:issue:`4178`, :issue:`4356`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
|
||||
returning ``None`` or modifying the ``params`` input parameter is now
|
||||
deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
|
||||
:meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
|
||||
:func:`scrapy.utils.request.request_from_dict` function.
|
||||
|
||||
- In :mod:`scrapy.squeues`, the following queue classes are deprecated:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
|
||||
instead use:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueue`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
|
||||
|
||||
- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
|
||||
a time when this class could handle multiple :class:`~scrapy.Spider`
|
||||
objects at a time have been deprecated. (:issue:`5090`)
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
|
||||
is deprecated.
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
|
||||
deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
|
||||
|
||||
- The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
|
||||
is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
|
||||
instead.
|
||||
|
||||
- The ``spider`` parameter is deprecated for the following methods:
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.download`
|
||||
|
||||
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
|
||||
first to set the :class:`~scrapy.Spider` object.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- You can now use :ref:`item filtering <item-filter>` to control which items
|
||||
are exported to each output feed. (:issue:`4575`, :issue:`5178`,
|
||||
:issue:`5161`, :issue:`5203`)
|
||||
|
||||
- You can now apply :ref:`post-processing <post-processing>` to feeds, and
|
||||
:ref:`built-in post-processing plugins <builtin-plugins>` are provided for
|
||||
output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
|
||||
|
||||
- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
|
||||
keys. (:issue:`5383`, :issue:`5384`)
|
||||
|
||||
- Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
|
||||
or later will automatically switch the asyncio event loop to one that
|
||||
allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
|
||||
:issue:`5315`)
|
||||
|
||||
- The :command:`genspider` command now supports a start URL instead of a
|
||||
domain name. (:issue:`4439`)
|
||||
|
||||
- :mod:`scrapy.utils.defer` gained 2 new functions,
|
||||
:func:`~scrapy.utils.defer.deferred_to_future` and
|
||||
:func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
|
||||
on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
|
||||
(:issue:`5288`)
|
||||
|
||||
- :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
|
||||
support for `temporary security credentials`_
|
||||
(:setting:`AWS_SESSION_TOKEN`) and endpoint customization
|
||||
(:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
|
||||
(:issue:`5279`)
|
||||
|
||||
- :attr:`Request.cookies <scrapy.Request.cookies>` values that are
|
||||
:class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
|
||||
(:issue:`5252`, :issue:`5253`)
|
||||
|
||||
- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
|
||||
the :signal:`spider_idle` signal to customize the reason why the spider is
|
||||
stopping. (:issue:`5191`)
|
||||
|
||||
- When using
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
|
||||
proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
|
||||
scheme. (:issue:`4505`, :issue:`4649`)
|
||||
|
||||
- All built-in queues now expose a ``peek`` method that returns the next
|
||||
queue object (like ``pop``) but does not remove the returned object from
|
||||
the queue. (:issue:`5112`)
|
||||
|
||||
If the underlying queue does not support peeking (e.g. because you are not
|
||||
using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
|
||||
:exc:`NotImplementedError`.
|
||||
|
||||
- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
|
||||
an ``attributes`` attribute that makes subclassing easier. For
|
||||
:class:`~scrapy.http.Request`, it also allows subclasses to work with
|
||||
:func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
|
||||
:issue:`5130`, :issue:`5218`)
|
||||
|
||||
- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
|
||||
:meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
|
||||
:ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
|
||||
|
||||
- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
|
||||
exceptions now only truncate response bodies longer than 1000 characters,
|
||||
instead of those longer than 32 characters, making it easier to debug such
|
||||
errors. (:issue:`4881`, :issue:`5007`)
|
||||
|
||||
- :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
|
||||
(:issue:`5145`, :issue:`5269`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
|
||||
are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
|
||||
(:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Removed a module-level Twisted reactor import that could prevent
|
||||
:ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
|
||||
|
||||
- The :command:`startproject` command works with existing folders again.
|
||||
(:issue:`4665`, :issue:`4676`)
|
||||
|
||||
- The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
|
||||
(:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
|
||||
``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
|
||||
|
||||
- Made :func:`scrapy.utils.response.open_in_browser` support more complex
|
||||
HTML. (:issue:`5319`, :issue:`5320`)
|
||||
|
||||
- Fixed :attr:`CSVFeedSpider.quotechar
|
||||
<scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
|
||||
encoding. (:issue:`5391`, :issue:`5394`)
|
||||
|
||||
- Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
|
||||
|
||||
.. _setuptools: https://pypi.org/project/setuptools/
|
||||
|
||||
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
now also works as expected with links that have comma-separated ``rel``
|
||||
attribute values including ``nofollow``. (:issue:`5225`)
|
||||
|
||||
- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
|
||||
<topics-feed-exports>` parameter parsing. (:issue:`5359`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental. (:issue:`5332`)
|
||||
|
||||
- Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
|
||||
(:issue:`4976`, :issue:`5315`)
|
||||
|
||||
- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
|
||||
(:issue:`4484`, :issue:`4613`)
|
||||
|
||||
- Documented :ref:`local file naming in media pipelines
|
||||
<topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
|
||||
|
||||
- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
|
||||
:issue:`3669`)
|
||||
|
||||
- Provided better context and instructions to disable the
|
||||
:setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
|
||||
|
||||
- Documented that :ref:`reppy-parser` does not support Python 3.9+.
|
||||
(:issue:`5226`, :issue:`5231`)
|
||||
|
||||
- Documented :ref:`the scheduler component <topics-scheduler>`.
|
||||
(:issue:`3537`, :issue:`3559`)
|
||||
|
||||
- Documented the method used by :ref:`media pipelines
|
||||
<topics-media-pipeline>` to :ref:`determine if a file has expired
|
||||
<file-expiration>`. (:issue:`5120`, :issue:`5254`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now features
|
||||
:func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now covers what happens when you define
|
||||
different per-spider values for some settings that cannot differ at run
|
||||
time. (:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Extended the documentation of the
|
||||
:class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
|
||||
(:issue:`5199`, :issue:`5217`)
|
||||
|
||||
- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
|
||||
:issue:`5224`)
|
||||
|
||||
- Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
|
||||
(:issue:`5174`, :issue:`5244`)
|
||||
|
||||
- Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
|
||||
(:issue:`1582`)
|
||||
|
||||
- Added the ``body_length`` parameter to the documented signature of the
|
||||
:signal:`headers_received` signal. (:issue:`5270`)
|
||||
|
||||
- Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
|
||||
in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
|
||||
|
||||
- The documentation now features the shortest import path of classes with
|
||||
multiple import paths. (:issue:`2733`, :issue:`5099`)
|
||||
|
||||
- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
|
||||
(:issue:`5395`, :issue:`5396`)
|
||||
|
||||
- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
|
||||
to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
|
||||
|
||||
- The pronunciation of the project name is now :ref:`officially
|
||||
<intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
|
||||
|
||||
- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
|
||||
|
||||
- Fixed issues and implemented minor improvements. (:issue:`3155`,
|
||||
:issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
|
||||
:issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
|
||||
:issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
|
||||
:issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
|
||||
|
||||
|
||||
Quality Assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
|
||||
:issue:`5265`)
|
||||
|
||||
- Significantly reduced memory usage by
|
||||
:func:`scrapy.utils.response.response_httprepr`, used by the
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
|
||||
middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
|
||||
|
||||
- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
|
||||
:issue:`5374`)
|
||||
|
||||
- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
|
||||
:issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
|
||||
|
||||
- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
|
||||
:issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
|
||||
:issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
|
||||
:issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
|
||||
:issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
|
||||
:issue:`5425`, :issue:`5427`)
|
||||
|
||||
- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
|
||||
:issue:`5177`, :issue:`5200`)
|
||||
|
||||
- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
|
||||
:issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
|
||||
:issue:`5314`, :issue:`5322`)
|
||||
|
||||
|
||||
.. _release-2.5.1:
|
||||
|
||||
Scrapy 2.5.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-2.5.0:
|
||||
|
||||
Scrapy 2.5.0 (2021-04-06)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
- Official Python 3.9 support
|
||||
|
||||
- Experimental :ref:`HTTP/2 support <http2>`
|
||||
|
||||
- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function
|
||||
to retry requests from spider callbacks
|
||||
|
||||
- New :class:`~scrapy.signals.headers_received` signal that allows stopping
|
||||
downloads early
|
||||
|
||||
- New :class:`Response.protocol <scrapy.http.Response.protocol>` attribute
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and
|
||||
had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`.
|
||||
(:issue:`4901`)
|
||||
|
||||
- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
|
||||
variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- The :mod:`scrapy.utils.py36` module is now deprecated in favor of
|
||||
:mod:`scrapy.utils.asyncgen`. (:issue:`4900`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Experimental :ref:`HTTP/2 support <http2>` through a new download handler
|
||||
that can be assigned to the ``https`` protocol in the
|
||||
:setting:`DOWNLOAD_HANDLERS` setting.
|
||||
(:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)
|
||||
|
||||
- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request`
|
||||
function may be used from spider callbacks or middlewares to handle the
|
||||
retrying of a request beyond the scenarios that
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports.
|
||||
(:issue:`3590`, :issue:`3685`, :issue:`4902`)
|
||||
|
||||
- The new :class:`~scrapy.signals.headers_received` signal gives early access
|
||||
to response headers and allows :ref:`stopping downloads
|
||||
<topics-stop-response-download>`.
|
||||
(:issue:`1772`, :issue:`4897`)
|
||||
|
||||
- The new :attr:`Response.protocol <scrapy.http.Response.protocol>`
|
||||
attribute gives access to the string that identifies the protocol used to
|
||||
download a response. (:issue:`4878`)
|
||||
|
||||
- :ref:`Stats <topics-stats>` now include the following entries that indicate
|
||||
the number of successes and failures in storing
|
||||
:ref:`feeds <topics-feed-exports>`::
|
||||
|
||||
feedexport/success_count/<storage type>
|
||||
feedexport/failed_count/<storage type>
|
||||
|
||||
Where ``<storage type>`` is the feed storage backend class name, such as
|
||||
:class:`~scrapy.extensions.feedexport.FileFeedStorage` or
|
||||
:class:`~scrapy.extensions.feedexport.FTPFeedStorage`.
|
||||
|
||||
(:issue:`3947`, :issue:`4850`)
|
||||
|
||||
- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider
|
||||
middleware now logs ignored URLs with ``INFO`` :ref:`logging level
|
||||
<levels>` instead of ``DEBUG``, and it now includes the following entry
|
||||
into :ref:`stats <topics-stats>` to keep track of the number of ignored
|
||||
URLs::
|
||||
|
||||
urllength/request_ignored_count
|
||||
|
||||
(:issue:`5036`)
|
||||
|
||||
- The
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
||||
downloader middleware now logs the number of decompressed responses and the
|
||||
total count of resulting bytes::
|
||||
|
||||
httpcompression/response_bytes
|
||||
httpcompression/response_count
|
||||
|
||||
(:issue:`4797`, :issue:`4799`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Fixed installation on PyPy installing PyDispatcher in addition to
|
||||
PyPyDispatcher, which could prevent Scrapy from working depending on which
|
||||
package got imported. (:issue:`4710`, :issue:`4814`)
|
||||
|
||||
- When inspecting a callback to check if it is a generator that also returns
|
||||
a value, an exception is no longer raised if the callback has a docstring
|
||||
with lower indentation than the following code.
|
||||
(:issue:`4477`, :issue:`4935`)
|
||||
|
||||
- The `Content-Length <https://tools.ietf.org/html/rfc2616#section-14.13>`_
|
||||
header is no longer omitted from responses when using the default, HTTP/1.1
|
||||
download handler (see :setting:`DOWNLOAD_HANDLERS`).
|
||||
(:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`)
|
||||
|
||||
- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False``
|
||||
now has the same effect as not setting it at all, instead of having the
|
||||
same effect as setting it to ``True``.
|
||||
(:issue:`3851`, :issue:`4694`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Added instructions to :ref:`install Scrapy in Windows using pip
|
||||
<intro-install-windows>`.
|
||||
(:issue:`4715`, :issue:`4736`)
|
||||
|
||||
- Logging documentation now includes :ref:`additional ways to filter logs
|
||||
<topics-logging-advanced-customization>`.
|
||||
(:issue:`4216`, :issue:`4257`, :issue:`4965`)
|
||||
|
||||
- Covered how to deal with long lists of allowed domains in the :ref:`FAQ
|
||||
<faq>`. (:issue:`2263`, :issue:`3667`)
|
||||
|
||||
- Covered scrapy-bench_ in :ref:`benchmarking`.
|
||||
(:issue:`4996`, :issue:`5016`)
|
||||
|
||||
- Clarified that one :ref:`extension <topics-extensions>` instance is created
|
||||
per crawler.
|
||||
(:issue:`5014`)
|
||||
|
||||
- Fixed some errors in examples.
|
||||
(:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`,
|
||||
:issue:`5008`)
|
||||
|
||||
- Fixed some external links, typos, and so on.
|
||||
(:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`,
|
||||
:issue:`5063`)
|
||||
|
||||
- The :ref:`list of Request.meta keys <topics-request-meta>` is now sorted
|
||||
alphabetically.
|
||||
(:issue:`5061`, :issue:`5065`)
|
||||
|
||||
- Updated references to Scrapinghub, which is now called Zyte.
|
||||
(:issue:`4973`, :issue:`5072`)
|
||||
|
||||
- Added a mention to contributors in the README. (:issue:`4956`)
|
||||
|
||||
- Reduced the top margin of lists. (:issue:`4974`)
|
||||
|
||||
|
||||
Quality Assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`)
|
||||
|
||||
- Extended typing hints (:issue:`4895`)
|
||||
|
||||
- Fixed deprecated uses of the Twisted API.
|
||||
(:issue:`4940`, :issue:`4950`, :issue:`5073`)
|
||||
|
||||
- Made our tests run with the new pip resolver.
|
||||
(:issue:`4710`, :issue:`4814`)
|
||||
|
||||
- Added tests to ensure that :ref:`coroutine support <coroutine-support>`
|
||||
is tested. (:issue:`4987`)
|
||||
|
||||
- Migrated from Travis CI to GitHub Actions. (:issue:`4924`)
|
||||
|
||||
- Fixed CI issues.
|
||||
(:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`,
|
||||
:issue:`5053`)
|
||||
|
||||
- Implemented code refactorings, style fixes and cleanups.
|
||||
(:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`)
|
||||
|
||||
|
||||
.. _release-2.4.1:
|
||||
|
||||
Scrapy 2.4.1 (2020-11-17)
|
||||
|
|
@ -97,6 +725,8 @@ Backward-incompatible changes
|
|||
(:issue:`4717`, :issue:`4823`)
|
||||
|
||||
|
||||
.. _2.4-deprecation-removals:
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -754,9 +1384,8 @@ Bug fixes
|
|||
* zope.interface 5.0.0 and later versions are now supported
|
||||
(:issue:`4447`, :issue:`4448`)
|
||||
|
||||
* :meth:`Spider.make_requests_from_url
|
||||
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
|
||||
1.4.0, now issues a warning when used (:issue:`4412`)
|
||||
* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
|
||||
warning when used (:issue:`4412`)
|
||||
|
||||
|
||||
Documentation
|
||||
|
|
@ -1014,7 +1643,7 @@ New features
|
|||
:issue:`4370`)
|
||||
|
||||
* A new ``keep_fragments`` parameter of
|
||||
:func:`scrapy.utils.request.request_fingerprint` allows to generate
|
||||
``scrapy.utils.request.request_fingerprint`` allows to generate
|
||||
different fingerprints for requests with different fragments in their URL
|
||||
(:issue:`4104`)
|
||||
|
||||
|
|
@ -1268,6 +1897,88 @@ affect subclasses:
|
|||
(:issue:`3884`)
|
||||
|
||||
|
||||
.. _release-1.8.2:
|
||||
|
||||
Scrapy 1.8.2 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
**Security bug fixes:**
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies into
|
||||
your requests to some other domains. Please, see the `mfjm-vh54-3f96
|
||||
security advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
.. _release-1.8.1:
|
||||
|
||||
Scrapy 1.8.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
|
|
@ -1433,6 +2144,8 @@ Deprecation removals
|
|||
* ``scrapy.xlib`` has been removed (:issue:`4015`)
|
||||
|
||||
|
||||
.. _1.8-deprecations:
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -1566,7 +2279,7 @@ New features
|
|||
* A new scheduler priority queue,
|
||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targetting multiple web domains, at the
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
|
||||
|
|
@ -1789,6 +2502,8 @@ The following deprecated settings have also been removed (:issue:`3578`):
|
|||
* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
|
||||
|
||||
|
||||
.. _1.7-deprecations:
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -2428,7 +3143,7 @@ Bug fixes
|
|||
- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
|
||||
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
|
||||
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
|
||||
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
|
||||
``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
|
@ -2602,7 +3317,7 @@ Bug fixes
|
|||
- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
|
||||
(:issue:`2225`).
|
||||
- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
|
||||
- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
|
||||
Refactoring
|
||||
~~~~~~~~~~~
|
||||
|
|
@ -3465,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09)
|
|||
- adding some xpath tips to selectors docs (:commit:`2d103e0`)
|
||||
- fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`)
|
||||
- get_func_args maximum recursion fix #728 (:commit:`81344ea`)
|
||||
- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Updated input/output processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Fixed Python syntax in tutorial. (:commit:`db59ed9`)
|
||||
- Add test case for tunneling proxy (:commit:`f090260`)
|
||||
- Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`)
|
||||
|
|
@ -4092,7 +4807,7 @@ Code rearranged and removed
|
|||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
|
|
@ -4127,7 +4842,7 @@ Scrapyd changes
|
|||
~~~~~~~~~~~~~~~
|
||||
|
||||
- Scrapyd now uses one process per spider
|
||||
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
|
||||
- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default)
|
||||
- A minimal web ui was added, available at http://localhost:6800 by default
|
||||
- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
|
||||
|
||||
|
|
@ -4163,7 +4878,7 @@ New features and improvements
|
|||
- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
|
||||
- Support for overriding default request headers per spider (#181)
|
||||
- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
|
||||
- Splitted Debian package into two packages - the library and the service (#187)
|
||||
- Split Debian package into two packages - the library and the service (#187)
|
||||
- Scrapy log refactoring (#188)
|
||||
- New extension for keeping persistent spider contexts among different runs (#203)
|
||||
- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
|
||||
|
|
@ -4184,7 +4899,7 @@ API changes
|
|||
- ``url`` and ``body`` attributes of Request objects are now read-only (#230)
|
||||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
|
|
@ -4331,6 +5046,7 @@ First release of Scrapy.
|
|||
.. _resource: https://docs.python.org/2/library/resource.html
|
||||
.. _robots.txt: https://www.robotstxt.org/
|
||||
.. _scrapely: https://github.com/scrapy/scrapely
|
||||
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
|
||||
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
|
||||
.. _six: https://six.readthedocs.io/
|
||||
.. _tox: https://pypi.org/project/tox/
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Sphinx>=3.0
|
||||
sphinx-hoverxref>=0.2b1
|
||||
sphinx-notfound-page>=0.4
|
||||
sphinx_rtd_theme>=0.4
|
||||
sphinx-rtd-theme>=0.5.2
|
||||
|
|
@ -29,9 +29,16 @@ how you :ref:`configure the downloader middlewares
|
|||
.. class:: Crawler(spidercls, settings)
|
||||
|
||||
The Crawler object must be instantiated with a
|
||||
:class:`scrapy.spiders.Spider` subclass and a
|
||||
:class:`scrapy.Spider` subclass and a
|
||||
:class:`scrapy.settings.Settings` object.
|
||||
|
||||
.. attribute:: request_fingerprinter
|
||||
|
||||
The request fingerprint builder of this crawler.
|
||||
|
||||
This is used from extensions and middlewares to build short, unique
|
||||
identifiers for requests. See :ref:`request-fingerprints`.
|
||||
|
||||
.. attribute:: settings
|
||||
|
||||
The settings manager of this crawler.
|
||||
|
|
@ -196,7 +203,7 @@ SpiderLoader API
|
|||
match the request's url against the domains of the spiders.
|
||||
|
||||
:param request: queried request
|
||||
:type request: :class:`~scrapy.http.Request` instance
|
||||
:type request: :class:`~scrapy.Request` instance
|
||||
|
||||
.. _topics-api-signals:
|
||||
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ this:
|
|||
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
|
||||
to crawl.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
9. The process repeats (from step 3) until there are no more requests from the
|
||||
:ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
Components
|
||||
|
|
@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the
|
|||
Scheduler
|
||||
---------
|
||||
|
||||
The Scheduler receives requests from the engine and enqueues them for feeding
|
||||
them later (also to the engine) when the engine requests them.
|
||||
The :ref:`scheduler <topics-scheduler>` receives requests from the engine and
|
||||
enqueues them for feeding them later (also to the engine) when the engine
|
||||
requests them.
|
||||
|
||||
.. _component-downloader:
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,10 @@ asyncio
|
|||
|
||||
.. versionadded:: 2.0
|
||||
|
||||
Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
|
||||
reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
||||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
|
||||
versions may introduce related changes without a deprecation
|
||||
period or warning.
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
|
|
@ -29,6 +26,7 @@ reactor manually. You can do that using
|
|||
|
||||
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
|
||||
|
||||
|
||||
.. _using-custom-loops:
|
||||
|
||||
Using custom asyncio loops
|
||||
|
|
@ -39,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
|
|||
use it instead of the default asyncio event loop.
|
||||
|
||||
|
||||
.. _asyncio-windows:
|
||||
|
||||
Windows-specific notes
|
||||
======================
|
||||
|
||||
The Windows implementation of :mod:`asyncio` can use two event loop
|
||||
implementations:
|
||||
|
||||
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
|
||||
when using Twisted.
|
||||
|
||||
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
|
||||
with Twisted.
|
||||
|
||||
So on Python 3.8+ the event loop class needs to be changed.
|
||||
|
||||
.. versionchanged:: 2.6.0
|
||||
The event loop class is changed automatically when you change the
|
||||
:setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
To change the event loop class manually, call the following code before
|
||||
installing the reactor::
|
||||
|
||||
import asyncio
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
You can put this in the same function that installs the reactor, if you do that
|
||||
yourself, or in some code that runs before the reactor is installed, e.g.
|
||||
``settings.py``.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
subprocesses (this is the case with `playwright`_), so you cannot use
|
||||
them together with Scrapy on Windows (but you should be able to use
|
||||
them on WSL or native Linux).
|
||||
|
||||
.. _playwright: https://github.com/microsoft/playwright-python
|
||||
|
||||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
Awaiting on Deferreds
|
||||
=====================
|
||||
|
||||
When the asyncio reactor isn't installed, you can await on Deferreds in the
|
||||
coroutines directly. When it is installed, this is not possible anymore, due to
|
||||
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
|
||||
:class:`asyncio.Future` objects, not into
|
||||
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
|
||||
Futures. Scrapy provides two helpers for this:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_to_future
|
||||
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
|
||||
.. tip:: If you need to use these functions in code that aims to be compatible
|
||||
with lower versions of Scrapy that do not provide these functions,
|
||||
down to Scrapy 2.0 (earlier versions do not support
|
||||
:mod:`asyncio`), you can copy the implementation of these functions
|
||||
into your own code.
|
||||
|
|
|
|||
|
|
@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
|
|||
results in slower crawl rates. How slower depends on how much your spider does
|
||||
and how well it's written.
|
||||
|
||||
In the future, more cases will be added to the benchmarking suite to cover
|
||||
other common scenarios.
|
||||
Use scrapy-bench_ for more complex benchmarking.
|
||||
|
||||
.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
|
||||
|
|
@ -230,10 +230,16 @@ Usage example::
|
|||
genspider
|
||||
---------
|
||||
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
|
||||
* Requires project: *no*
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
.. versionadded:: 2.6.0
|
||||
The ability to pass a URL instead of a domain.
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
|
||||
.. note:: Even if an HTTPS URL is specified, the protocol used in
|
||||
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
|
@ -598,8 +604,6 @@ Example:
|
|||
Register commands via setup.py entry points
|
||||
-------------------------------------------
|
||||
|
||||
.. note:: This is an experimental feature, use with caution.
|
||||
|
||||
You can also add Scrapy commands from an external library by adding a
|
||||
``scrapy.commands`` section in the entry points of the library ``setup.py``
|
||||
file.
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ This callback is tested using three built-in contracts:
|
|||
|
||||
.. class:: CallbackKeywordArgumentsContract
|
||||
|
||||
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
|
||||
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
|
||||
attribute for the sample request. It must be a valid JSON dictionary.
|
||||
::
|
||||
|
||||
|
|
@ -88,7 +88,7 @@ override three methods:
|
|||
.. method:: Contract.adjust_request_args(args)
|
||||
|
||||
This receives a ``dict`` as an argument containing default arguments
|
||||
for request object. :class:`~scrapy.http.Request` is used by default,
|
||||
for request object. :class:`~scrapy.Request` is used by default,
|
||||
but this can be changed with the ``request_cls`` attribute.
|
||||
If multiple contracts in chain have this attribute defined, the last one is used.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. _topics-coroutines:
|
||||
|
||||
==========
|
||||
Coroutines
|
||||
==========
|
||||
|
|
@ -15,7 +17,7 @@ Supported callables
|
|||
The following callables may be defined as coroutines using ``async def``, and
|
||||
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
||||
|
||||
- :class:`~scrapy.http.Request` callbacks.
|
||||
- :class:`~scrapy.Request` callbacks.
|
||||
|
||||
.. note:: The callback output is not processed until the whole callback
|
||||
finishes.
|
||||
|
|
@ -75,23 +77,28 @@ coroutines, functions that return Deferreds and functions that return
|
|||
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
|
||||
This means you can use many useful Python libraries providing such code::
|
||||
|
||||
class MySpider(Spider):
|
||||
class MySpiderDeferred(Spider):
|
||||
# ...
|
||||
async def parse_with_deferred(self, response):
|
||||
async def parse(self, response):
|
||||
additional_response = await treq.get('https://additional.url')
|
||||
additional_data = await treq.content(additional_response)
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
async def parse_with_asyncio(self, response):
|
||||
class MySpiderAsyncio(Spider):
|
||||
# ...
|
||||
async def parse(self, response):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get('https://additional.url') as additional_response:
|
||||
additional_data = await r.text()
|
||||
additional_data = await additional_response.text()
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
|
||||
:mod:`asyncio` loop and to use them you need to
|
||||
:doc:`enable asyncio support in Scrapy<asyncio>`.
|
||||
|
||||
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
|
||||
you need to :ref:`wrap them<asyncio-await-dfd>`.
|
||||
|
||||
Common use cases for asynchronous code include:
|
||||
|
||||
* requesting data from websites, databases and other services (in callbacks,
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Consider the following Scrapy spider below::
|
|||
|
||||
Basically this is a simple spider which parses two pages of items (the
|
||||
start_urls). Items also have a details page with additional information, so we
|
||||
use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
|
||||
use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a
|
||||
partially populated item.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ spiders come in.
|
|||
Popular choices for deploying Scrapy spiders are:
|
||||
|
||||
* :ref:`Scrapyd <deploy-scrapyd>` (open source)
|
||||
* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
|
||||
* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
|
||||
|
||||
.. _deploy-scrapyd:
|
||||
|
||||
|
|
@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
|
|||
|
||||
.. _deploy-scrapy-cloud:
|
||||
|
||||
Deploying to Scrapy Cloud
|
||||
=========================
|
||||
Deploying to Zyte Scrapy Cloud
|
||||
==============================
|
||||
|
||||
`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
|
||||
the company behind Scrapy.
|
||||
`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
|
||||
behind Scrapy.
|
||||
|
||||
Scrapy Cloud removes the need to setup and monitor servers
|
||||
and provides a nice UI to manage spiders and review scraped items,
|
||||
logs and stats.
|
||||
Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
|
||||
nice UI to manage spiders and review scraped items, logs and stats.
|
||||
|
||||
To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
|
||||
Please refer to the `Scrapy Cloud documentation`_ for more information.
|
||||
To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
|
||||
tool.
|
||||
Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
|
||||
|
||||
Scrapy Cloud is compatible with Scrapyd and one can switch between
|
||||
Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
|
||||
them as needed - the configuration is read from the ``scrapy.cfg`` file
|
||||
just like ``scrapyd-deploy``.
|
||||
|
||||
.. _Scrapyd: https://github.com/scrapy/scrapyd
|
||||
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
|
||||
.. _Scrapyd: https://github.com/scrapy/scrapyd
|
||||
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
|
||||
.. _shub: https://doc.scrapinghub.com/shub.html
|
||||
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
|
||||
.. _Scrapinghub: https://scrapinghub.com/
|
||||
.. _shub: https://shub.readthedocs.io/en/latest/
|
||||
.. _Zyte: https://zyte.com/
|
||||
.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
|
||||
.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
|
|||
|
||||
Since Developer Tools operate on a live browser DOM, what you'll actually see
|
||||
when inspecting the page source is not the original HTML, but a modified one
|
||||
after applying some browser clean up and executing Javascript code. Firefox,
|
||||
after applying some browser clean up and executing JavaScript code. Firefox,
|
||||
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
|
||||
the other hand, does not modify the original page HTML, so you won't be able to
|
||||
extract any data if you use ``<tbody>`` in your XPath expressions.
|
||||
|
||||
Therefore, you should keep in mind the following things:
|
||||
|
||||
* Disable Javascript while inspecting the DOM looking for XPaths to be
|
||||
* Disable JavaScript while inspecting the DOM looking for XPaths to be
|
||||
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
|
||||
|
||||
* Never use full XPath paths, use relative and clever ones based on attributes
|
||||
|
|
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
|
|||
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
|
||||
copy XPaths to selected elements. Let's try it out.
|
||||
|
||||
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
|
||||
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
$ scrapy shell "http://quotes.toscrape.com/"
|
||||
$ scrapy shell "https://quotes.toscrape.com/"
|
||||
|
||||
Then, back to your web browser, right-click on the ``span`` tag, select
|
||||
``Copy > XPath`` and paste it in the Scrapy shell like so:
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
|
||||
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
|
||||
|
||||
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
|
||||
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
|
||||
|
|
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
|
|||
type ``json``.
|
||||
|
||||
If we click on this request, we see that the request URL is
|
||||
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
is a JSON-object that contains our quotes. We can also right-click
|
||||
on the request and open ``Open in new tab`` to get a better overview.
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
|
|||
name = 'quote'
|
||||
allowed_domains = ['quotes.toscrape.com']
|
||||
page = 1
|
||||
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
|
||||
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
|
||||
|
||||
def parse(self, response):
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
|
|||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
self.page += 1
|
||||
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
This spider starts at the first page of the quotes-API. With each
|
||||
|
|
@ -274,13 +274,13 @@ In more complex websites, it could be difficult to easily reproduce the
|
|||
requests, as we could need to add ``headers`` or ``cookies`` to make it work.
|
||||
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
|
||||
format, by right-clicking on each of them in the network tool and using the
|
||||
:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
|
||||
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
|
||||
request::
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
request = Request.from_curl(
|
||||
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
|
||||
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
|
||||
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
|
||||
|
|
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
|
|||
to identifying the correct request and replicating it in your spider.
|
||||
|
||||
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
|
||||
.. _quotes.toscrape.com: http://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _quotes.toscrape.com: https://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
|
||||
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
middleware.
|
||||
|
||||
:meth:`process_request` should either: return ``None``, return a
|
||||
:class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request`
|
||||
:class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request`
|
||||
object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this request, executing all
|
||||
|
|
@ -88,8 +88,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
or the appropriate download function; it'll return that response. The :meth:`process_response`
|
||||
methods of installed middleware is always called on every response.
|
||||
|
||||
If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling
|
||||
process_request methods and reschedule the returned request. Once the newly returned
|
||||
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
|
||||
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
|
||||
request is performed, the appropriate middleware chain will be called on
|
||||
the downloaded response.
|
||||
|
||||
|
|
@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
ignored and not logged (unlike other exceptions).
|
||||
|
||||
:param request: the request being processed
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider for which this request is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: process_response(request, response, spider)
|
||||
|
||||
:meth:`process_response` should either: return a :class:`~scrapy.http.Response`
|
||||
object, return a :class:`~scrapy.http.Request` object or
|
||||
object, return a :class:`~scrapy.Request` object or
|
||||
raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception.
|
||||
|
||||
If it returns a :class:`~scrapy.http.Response` (it could be the same given
|
||||
response, or a brand-new one), that response will continue to be processed
|
||||
with the :meth:`process_response` of the next middleware in the chain.
|
||||
|
||||
If it returns a :class:`~scrapy.http.Request` object, the middleware chain is
|
||||
If it returns a :class:`~scrapy.Request` object, the middleware chain is
|
||||
halted and the returned request is rescheduled to be downloaded in the future.
|
||||
This is the same behavior as if a request is returned from :meth:`process_request`.
|
||||
|
||||
|
|
@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
exception, it is ignored and not logged (unlike other exceptions).
|
||||
|
||||
:param request: the request that originated the response
|
||||
:type request: is a :class:`~scrapy.http.Request` object
|
||||
:type request: is a :class:`~scrapy.Request` object
|
||||
|
||||
:param response: the response being processed
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider for which this response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: process_exception(request, exception, spider)
|
||||
|
||||
|
|
@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)
|
||||
|
||||
:meth:`process_exception` should return: either ``None``,
|
||||
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object.
|
||||
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
executing any other :meth:`process_exception` methods of installed middleware,
|
||||
|
|
@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
method chain of installed middleware is started, and Scrapy won't bother calling
|
||||
any other :meth:`process_exception` methods of middleware.
|
||||
|
||||
If it returns a :class:`~scrapy.http.Request` object, the returned request is
|
||||
If it returns a :class:`~scrapy.Request` object, the returned request is
|
||||
rescheduled to be downloaded in the future. This stops the execution of
|
||||
:meth:`process_exception` methods of the middleware the same as returning a
|
||||
response would.
|
||||
|
||||
:param request: the request that generated the exception
|
||||
:type request: is a :class:`~scrapy.http.Request` object
|
||||
:type request: is a :class:`~scrapy.Request` object
|
||||
|
||||
:param exception: the raised exception
|
||||
:type exception: an ``Exception`` object
|
||||
|
||||
:param spider: the spider for which this request is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
|
|
@ -203,13 +203,13 @@ CookiesMiddleware
|
|||
browsers do.
|
||||
|
||||
.. caution:: When non-UTF8 encoded byte sequences are passed to a
|
||||
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
|
||||
:class:`~scrapy.Request`, the ``CookiesMiddleware`` will log
|
||||
a warning. Refer to :ref:`topics-logging-advanced-customization`
|
||||
to customize the logging behaviour.
|
||||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
The following settings can be used to configure the cookie middleware:
|
||||
|
|
@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will
|
|||
**not** be merged with the existing cookies.
|
||||
|
||||
For more detailed information see the ``cookies`` parameter in
|
||||
:class:`~scrapy.http.Request`.
|
||||
:class:`~scrapy.Request`.
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
|
|
@ -323,8 +323,21 @@ HttpAuthMiddleware
|
|||
This middleware authenticates all requests generated from certain spiders
|
||||
using `Basic access authentication`_ (aka. HTTP auth).
|
||||
|
||||
To enable HTTP authentication from certain spiders, set the ``http_user``
|
||||
and ``http_pass`` attributes of those spiders.
|
||||
To enable HTTP authentication for a spider, set the ``http_user`` and
|
||||
``http_pass`` spider attributes to the authentication data and the
|
||||
``http_auth_domain`` spider attribute to the domain which requires this
|
||||
authentication (its subdomains will be also handled in the same way).
|
||||
You can set ``http_auth_domain`` to ``None`` to enable the
|
||||
authentication for all requests but you risk leaking your authentication
|
||||
credentials to unrelated domains.
|
||||
|
||||
.. warning::
|
||||
In previous Scrapy versions HttpAuthMiddleware sent the authentication
|
||||
data with all requests, which is a security problem if the spider
|
||||
makes requests to several different domains. Currently if the
|
||||
``http_auth_domain`` attribute is not set, the middleware will use the
|
||||
domain of the first request, which will work for some spiders but not
|
||||
for others. In the future the middleware will produce an error instead.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -334,6 +347,7 @@ HttpAuthMiddleware
|
|||
|
||||
http_user = 'someuser'
|
||||
http_pass = 'somepass'
|
||||
http_auth_domain = 'intranet.example.com'
|
||||
name = 'intranet.example.com'
|
||||
|
||||
# .. rest of the spider code omitted ...
|
||||
|
|
@ -352,7 +366,7 @@ HttpCacheMiddleware
|
|||
This middleware provides low-level cache to all HTTP requests and responses.
|
||||
It has to be combined with a cache storage backend as well as a cache policy.
|
||||
|
||||
Scrapy ships with three HTTP cache storage backends:
|
||||
Scrapy ships with the following HTTP cache storage backends:
|
||||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
|
|
@ -501,7 +515,7 @@ defines the methods described below.
|
|||
the :signal:`open_spider <spider_opened>` signal.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: close_spider(spider)
|
||||
|
||||
|
|
@ -509,27 +523,27 @@ defines the methods described below.
|
|||
the :signal:`close_spider <spider_closed>` signal.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: retrieve_response(spider, request)
|
||||
|
||||
Return response if present in cache, or ``None`` otherwise.
|
||||
|
||||
:param spider: the spider which generated the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param request: the request to find cached response for
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
.. method:: store_response(spider, request, response)
|
||||
|
||||
Store the given response in the cache.
|
||||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param request: the corresponding request the spider generated
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param response: the response to store in the cache
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -690,14 +704,15 @@ HttpCompressionMiddleware
|
|||
sent/received from web sites.
|
||||
|
||||
This middleware also supports decoding `brotli-compressed`_ as well as
|
||||
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
|
||||
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
|
||||
installed, respectively.
|
||||
|
||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||
.. _brotlipy: https://pypi.org/project/brotlipy/
|
||||
.. _brotli: https://pypi.org/project/Brotli/
|
||||
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
||||
.. _zstandard: https://pypi.org/project/zstandard/
|
||||
|
||||
|
||||
HttpCompressionMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -722,7 +737,7 @@ HttpProxyMiddleware
|
|||
.. class:: HttpProxyMiddleware
|
||||
|
||||
This middleware sets the HTTP proxy to use for requests, by setting the
|
||||
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
|
||||
``proxy`` meta value for :class:`~scrapy.Request` objects.
|
||||
|
||||
Like the Python standard library module :mod:`urllib.request`, it obeys
|
||||
the following environment variables:
|
||||
|
|
@ -749,12 +764,12 @@ RedirectMiddleware
|
|||
.. reqmeta:: redirect_urls
|
||||
|
||||
The urls which the request goes through (while being redirected) can be found
|
||||
in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
|
||||
in the ``redirect_urls`` :attr:`Request.meta <scrapy.Request.meta>` key.
|
||||
|
||||
.. reqmeta:: redirect_reasons
|
||||
|
||||
The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
|
||||
``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
|
||||
``redirect_reasons`` :attr:`Request.meta <scrapy.Request.meta>` key. For
|
||||
example: ``[301, 302, 307, 'meta refresh']``.
|
||||
|
||||
The format of a reason depends on the middleware that handled the corresponding
|
||||
|
|
@ -770,7 +785,7 @@ settings (see the settings documentation for more info):
|
|||
|
||||
.. reqmeta:: dont_redirect
|
||||
|
||||
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_redirect``
|
||||
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``
|
||||
key set to True, the request will be ignored by this middleware.
|
||||
|
||||
If you want to handle some redirect status codes in your spider, you can
|
||||
|
|
@ -783,7 +798,7 @@ responses (and pass them through to your spider) you can do this::
|
|||
handle_httpstatus_list = [301, 302]
|
||||
|
||||
The ``handle_httpstatus_list`` key of :attr:`Request.meta
|
||||
<scrapy.http.Request.meta>` can also be used to specify which response codes to
|
||||
<scrapy.Request.meta>` can also be used to specify which response codes to
|
||||
allow on a per-request basis. You can also set the meta key
|
||||
``handle_httpstatus_all`` to ``True`` if you want to allow any response code
|
||||
for a request.
|
||||
|
|
@ -889,9 +904,14 @@ settings (see the settings documentation for more info):
|
|||
|
||||
.. reqmeta:: dont_retry
|
||||
|
||||
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
|
||||
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
|
||||
set to True, the request will be ignored by this middleware.
|
||||
|
||||
To retry requests from a spider callback, you can use the
|
||||
:func:`get_retry_request` function:
|
||||
|
||||
.. autofunction:: get_retry_request
|
||||
|
||||
RetryMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -914,7 +934,7 @@ Default: ``2``
|
|||
Maximum number of times to retry, in addition to the first download.
|
||||
|
||||
Maximum number of retries can also be specified per-request using
|
||||
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
|
||||
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.Request.meta>`.
|
||||
When initialized, the :reqmeta:`max_retry_times` meta key takes higher
|
||||
precedence over the :setting:`RETRY_TIMES` setting.
|
||||
|
||||
|
|
@ -932,6 +952,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
|
|||
it is a common code used to indicate server overload. It is not included by
|
||||
default because HTTP specs say so.
|
||||
|
||||
.. setting:: RETRY_PRIORITY_ADJUST
|
||||
|
||||
RETRY_PRIORITY_ADJUST
|
||||
---------------------
|
||||
|
||||
Default: ``-1``
|
||||
|
||||
Adjust retry request priority relative to original request:
|
||||
|
||||
- a positive priority adjust means higher priority.
|
||||
- **a negative priority adjust (default) means lower priority.**
|
||||
|
||||
|
||||
.. _topics-dlmw-robots:
|
||||
|
||||
|
|
@ -969,7 +1001,7 @@ RobotsTxtMiddleware
|
|||
|
||||
.. reqmeta:: dont_obey_robotstxt
|
||||
|
||||
If :attr:`Request.meta <scrapy.http.Request.meta>` has
|
||||
If :attr:`Request.meta <scrapy.Request.meta>` has
|
||||
``dont_obey_robotstxt`` key set to True
|
||||
the request will be ignored by this middleware even if
|
||||
:setting:`ROBOTSTXT_OBEY` is enabled.
|
||||
|
|
@ -988,7 +1020,7 @@ Parsers vary in several aspects:
|
|||
(shorter) rule
|
||||
|
||||
Performance comparison of different parsers is available at `the following link
|
||||
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
|
||||
<https://github.com/scrapy/scrapy/issues/3969>`_.
|
||||
|
||||
.. _protego-parser:
|
||||
|
||||
|
|
@ -1053,9 +1085,13 @@ In order to use this parser:
|
|||
|
||||
* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
|
||||
|
||||
.. warning:: `Upstream issue #122
|
||||
<https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
|
||||
|
||||
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
||||
``scrapy.robotstxt.ReppyRobotParser``
|
||||
|
||||
|
||||
.. _rerp-parser:
|
||||
|
||||
Robotexclusionrulesparser
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the
|
|||
information can be found in the response they get.
|
||||
|
||||
If they get a response with the desired data, modify your Scrapy
|
||||
:class:`~scrapy.http.Request` to match that of the other HTTP client. For
|
||||
:class:`~scrapy.Request` to match that of the other HTTP client. For
|
||||
example, try using the same user-agent string (:setting:`USER_AGENT`) or the
|
||||
same :attr:`~scrapy.http.Request.headers`.
|
||||
same :attr:`~scrapy.Request.headers`.
|
||||
|
||||
If they also get a response without the desired data, you’ll need to take
|
||||
steps to make your request more similar to that of the web browser. See
|
||||
|
|
@ -81,14 +81,14 @@ Use the :ref:`network tool <topics-network-tool>` of your web browser to see
|
|||
how your web browser performs the desired request, and try to reproduce that
|
||||
request with Scrapy.
|
||||
|
||||
It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
|
||||
It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
|
||||
method and URL. However, you may also need to reproduce the body, headers and
|
||||
form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
|
||||
form parameters (see :class:`~scrapy.FormRequest`) of that request.
|
||||
|
||||
As all major browsers allow to export the requests in `cURL
|
||||
<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
|
||||
:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
|
||||
:class:`~scrapy.http.Request` from a cURL command. To get more information
|
||||
:meth:`~scrapy.Request.from_curl()` to generate an equivalent
|
||||
:class:`~scrapy.Request` from a cURL command. To get more information
|
||||
visit :ref:`request from curl <requests-from-curl>` inside the network
|
||||
tool section.
|
||||
|
||||
|
|
@ -125,7 +125,7 @@ data from it depends on the type of response:
|
|||
|
||||
If the desired data is inside HTML or XML code embedded within JSON data,
|
||||
you can load that HTML or XML code into a
|
||||
:class:`~scrapy.selector.Selector` and then
|
||||
:class:`~scrapy.Selector` and then
|
||||
:ref:`use it <topics-selectors>` as usual::
|
||||
|
||||
selector = Selector(data['html'])
|
||||
|
|
@ -246,24 +246,46 @@ Using a headless browser
|
|||
========================
|
||||
|
||||
A `headless browser`_ is a special web browser that provides an API for
|
||||
automation.
|
||||
automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
|
||||
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
|
||||
|
||||
The easiest way to use a headless browser with Scrapy is to use Selenium_,
|
||||
along with `scrapy-selenium`_ for seamless integration.
|
||||
One such library is `playwright-python`_ (an official Python port of `playwright`_).
|
||||
The following is a simple snippet to illustrate its usage within a Scrapy spider::
|
||||
|
||||
import scrapy
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
class PlaywrightSpider(scrapy.Spider):
|
||||
name = "playwright"
|
||||
start_urls = ["data:,"] # avoid using the default Scrapy downloader
|
||||
|
||||
async def parse(self, response):
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch()
|
||||
page = await browser.new_page()
|
||||
await page.goto("https:/example.org")
|
||||
title = await page.title()
|
||||
return {"title": title}
|
||||
|
||||
|
||||
However, using `playwright-python`_ directly as in the above example
|
||||
circumvents most of the Scrapy components (middlewares, dupefilter, etc).
|
||||
We recommend using `scrapy-playwright`_ for a better integration.
|
||||
|
||||
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
|
||||
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
|
||||
.. _Splash: https://github.com/scrapinghub/splash
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
.. _curl: https://curl.haxx.se/
|
||||
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
|
||||
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
|
||||
.. _js2xml: https://github.com/scrapinghub/js2xml
|
||||
.. _playwright-python: https://github.com/microsoft/playwright-python
|
||||
.. _playwright: https://github.com/microsoft/playwright
|
||||
.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
|
||||
.. _pytesseract: https://github.com/madmaze/pytesseract
|
||||
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
|
||||
.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
.. _Selenium: https://www.selenium.dev/
|
||||
.. _Splash: https://github.com/scrapinghub/splash
|
||||
.. _tabula-py: https://github.com/chezou/tabula-py
|
||||
.. _wget: https://www.gnu.org/software/wget/
|
||||
.. _wgrep: https://github.com/stav/wgrep
|
||||
.. _wgrep: https://github.com/stav/wgrep
|
||||
|
|
|
|||
|
|
@ -64,10 +64,10 @@ NotConfigured
|
|||
This exception can be raised by some components to indicate that they will
|
||||
remain disabled. Those components include:
|
||||
|
||||
* Extensions
|
||||
* Item pipelines
|
||||
* Downloader middlewares
|
||||
* Spider middlewares
|
||||
- Extensions
|
||||
- Item pipelines
|
||||
- Downloader middlewares
|
||||
- Spider middlewares
|
||||
|
||||
The exception must be raised in the component's ``__init__`` method.
|
||||
|
||||
|
|
@ -85,8 +85,8 @@ StopDownload
|
|||
|
||||
.. exception:: StopDownload(fail=True)
|
||||
|
||||
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
|
||||
indicate that no further bytes should be downloaded for a response.
|
||||
Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
|
||||
signal handler to indicate that no further bytes should be downloaded for a response.
|
||||
|
||||
The ``fail`` boolean parameter controls which method will handle the resulting
|
||||
response:
|
||||
|
|
@ -110,5 +110,6 @@ attribute.
|
|||
``StopDownload(False)`` or ``StopDownload(True)`` will raise
|
||||
a :class:`TypeError`.
|
||||
|
||||
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
|
||||
See the documentation for the :class:`~scrapy.signals.bytes_received` and
|
||||
:class:`~scrapy.signals.headers_received` signals
|
||||
and the :ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
|
|
|
|||
|
|
@ -50,18 +50,19 @@ value of one of their fields::
|
|||
self.year_to_exporter = {}
|
||||
|
||||
def close_spider(self, spider):
|
||||
for exporter in self.year_to_exporter.values():
|
||||
for exporter, xml_file in self.year_to_exporter.values():
|
||||
exporter.finish_exporting()
|
||||
xml_file.close()
|
||||
|
||||
def _exporter_for_item(self, item):
|
||||
adapter = ItemAdapter(item)
|
||||
year = adapter['year']
|
||||
if year not in self.year_to_exporter:
|
||||
f = open(f'{year}.xml', 'wb')
|
||||
exporter = XmlItemExporter(f)
|
||||
xml_file = open(f'{year}.xml', 'wb')
|
||||
exporter = XmlItemExporter(xml_file)
|
||||
exporter.start_exporting()
|
||||
self.year_to_exporter[year] = exporter
|
||||
return self.year_to_exporter[year]
|
||||
self.year_to_exporter[year] = (exporter, xml_file)
|
||||
return self.year_to_exporter[year][0]
|
||||
|
||||
def process_item(self, item, spider):
|
||||
exporter = self._exporter_for_item(item)
|
||||
|
|
@ -89,7 +90,7 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
If you use :class:`~scrapy.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
|
|
@ -121,9 +122,9 @@ Example::
|
|||
class ProductXmlExporter(XmlItemExporter):
|
||||
|
||||
def serialize_field(self, field, name, value):
|
||||
if field == 'price':
|
||||
if name == 'price':
|
||||
return f'$ {str(value)}'
|
||||
return super(Product, self).serialize_field(field, name, value)
|
||||
return super().serialize_field(field, name, value)
|
||||
|
||||
.. _topics-exporters-reference:
|
||||
|
||||
|
|
@ -171,7 +172,7 @@ BaseItemExporter
|
|||
:param field: the field being serialized. If the source :ref:`item object
|
||||
<item-types>` does not define field metadata, *field* is an empty
|
||||
:class:`dict`.
|
||||
:type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
|
||||
:type field: :class:`~scrapy.Field` object or a :class:`dict` instance
|
||||
|
||||
:param name: the name of the field being serialized
|
||||
:type name: str
|
||||
|
|
|
|||
|
|
@ -7,8 +7,7 @@ Extensions
|
|||
The extensions framework provides a mechanism for inserting your own
|
||||
custom functionality into Scrapy.
|
||||
|
||||
Extensions are just regular classes that are instantiated at Scrapy startup,
|
||||
when extensions are initialized.
|
||||
Extensions are just regular classes.
|
||||
|
||||
Extension settings
|
||||
==================
|
||||
|
|
@ -27,8 +26,8 @@ Loading & activating extensions
|
|||
===============================
|
||||
|
||||
Extensions are loaded and activated at startup by instantiating a single
|
||||
instance of the extension class. Therefore, all the extension initialization
|
||||
code must be performed in the class ``__init__`` method.
|
||||
instance of the extension class per spider being run. All the extension
|
||||
initialization code must be performed in the class ``__init__`` method.
|
||||
|
||||
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
|
||||
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
|
||||
|
|
@ -324,6 +323,11 @@ domain has finished scraping, including the Scrapy stats collected. The email
|
|||
will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
|
||||
setting.
|
||||
|
||||
Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a
|
||||
full list of parameters, including examples on how to instantiate
|
||||
:class:`~scrapy.mail.MailSender` and use mail settings, see
|
||||
:ref:`topics-email`.
|
||||
|
||||
.. module:: scrapy.extensions.debug
|
||||
:synopsis: Extensions for debugging Scrapy
|
||||
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ Serialization formats
|
|||
For serializing the scraped data, the feed exports use the :ref:`Item exporters
|
||||
<topics-exporters>`. These formats are supported out of the box:
|
||||
|
||||
* :ref:`topics-feed-format-json`
|
||||
* :ref:`topics-feed-format-jsonlines`
|
||||
* :ref:`topics-feed-format-csv`
|
||||
* :ref:`topics-feed-format-xml`
|
||||
- :ref:`topics-feed-format-json`
|
||||
- :ref:`topics-feed-format-jsonlines`
|
||||
- :ref:`topics-feed-format-csv`
|
||||
- :ref:`topics-feed-format-xml`
|
||||
|
||||
But you can also extend the supported format through the
|
||||
:setting:`FEED_EXPORTERS` setting.
|
||||
|
|
@ -34,54 +34,58 @@ But you can also extend the supported format through the
|
|||
JSON
|
||||
----
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
|
||||
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
|
||||
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
|
||||
large feeds.
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
|
||||
|
||||
- Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
|
||||
|
||||
- See :ref:`this warning <json-with-large-data>` if you're using JSON with
|
||||
large feeds.
|
||||
|
||||
.. _topics-feed-format-jsonlines:
|
||||
|
||||
JSON lines
|
||||
----------
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
|
||||
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
|
||||
- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
|
||||
|
||||
.. _topics-feed-format-csv:
|
||||
|
||||
CSV
|
||||
---
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
|
||||
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
|
||||
* To specify columns to export, their order and their column names, use
|
||||
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
|
||||
option, but it is important for CSV because unlike many other export
|
||||
formats CSV uses a fixed header.
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
|
||||
|
||||
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
|
||||
|
||||
- To specify columns to export, their order and their column names, use
|
||||
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
|
||||
option, but it is important for CSV because unlike many other export
|
||||
formats CSV uses a fixed header.
|
||||
|
||||
.. _topics-feed-format-xml:
|
||||
|
||||
XML
|
||||
---
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
|
||||
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
|
||||
- Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
|
||||
|
||||
.. _topics-feed-format-pickle:
|
||||
|
||||
Pickle
|
||||
------
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
|
||||
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
|
||||
- Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
|
||||
|
||||
.. _topics-feed-format-marshal:
|
||||
|
||||
Marshal
|
||||
-------
|
||||
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
|
||||
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
|
||||
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
|
||||
|
||||
.. _topics-feed-storage:
|
||||
|
|
@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme.
|
|||
|
||||
The storages backends supported out of the box are:
|
||||
|
||||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
- :ref:`topics-feed-storage-fs`
|
||||
- :ref:`topics-feed-storage-ftp`
|
||||
- :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
|
||||
- :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
not available. For example, the S3 backend is only available if the botocore_
|
||||
|
|
@ -114,8 +118,8 @@ Storage URI parameters
|
|||
The storage URI can also contain parameters that get replaced when the feed is
|
||||
being created. These parameters are:
|
||||
|
||||
* ``%(time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
* ``%(name)s`` - gets replaced by the spider name
|
||||
- ``%(time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
- ``%(name)s`` - gets replaced by the spider name
|
||||
|
||||
Any other named parameter gets replaced by the spider attribute of the same
|
||||
name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
|
||||
|
|
@ -123,13 +127,16 @@ attribute the moment the feed is being created.
|
|||
|
||||
Here are some examples to illustrate:
|
||||
|
||||
* Store in FTP using one directory per spider:
|
||||
- Store in FTP using one directory per spider:
|
||||
|
||||
* ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
|
||||
- ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
|
||||
|
||||
* Store in S3 using one directory per spider:
|
||||
- Store in S3 using one directory per spider:
|
||||
|
||||
* ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
|
||||
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
|
||||
|
||||
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
|
||||
they can also be used as storage URI parameters.
|
||||
|
||||
|
||||
.. _topics-feed-storage-backends:
|
||||
|
|
@ -144,9 +151,9 @@ Local filesystem
|
|||
|
||||
The feeds are stored in the local filesystem.
|
||||
|
||||
* URI scheme: ``file``
|
||||
* Example URI: ``file:///tmp/export.csv``
|
||||
* Required external libraries: none
|
||||
- URI scheme: ``file``
|
||||
- Example URI: ``file:///tmp/export.csv``
|
||||
- Required external libraries: none
|
||||
|
||||
Note that for the local filesystem storage (only) you can omit the scheme if
|
||||
you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
|
||||
|
|
@ -159,9 +166,9 @@ FTP
|
|||
|
||||
The feeds are stored in a FTP server.
|
||||
|
||||
* URI scheme: ``ftp``
|
||||
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
|
||||
* Required external libraries: none
|
||||
- URI scheme: ``ftp``
|
||||
- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
|
||||
- Required external libraries: none
|
||||
|
||||
FTP supports two different connection modes: `active or passive
|
||||
<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
|
||||
|
|
@ -178,23 +185,29 @@ S3
|
|||
|
||||
The feeds are stored on `Amazon S3`_.
|
||||
|
||||
* URI scheme: ``s3``
|
||||
* Example URIs:
|
||||
- URI scheme: ``s3``
|
||||
|
||||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
- Example URIs:
|
||||
|
||||
* Required external libraries: `botocore`_ >= 1.4.87
|
||||
- ``s3://mybucket/path/to/export.csv``
|
||||
|
||||
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
- Required external libraries: `botocore`_ >= 1.4.87
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
||||
* :setting:`AWS_ACCESS_KEY_ID`
|
||||
* :setting:`AWS_SECRET_ACCESS_KEY`
|
||||
- :setting:`AWS_ACCESS_KEY_ID`
|
||||
- :setting:`AWS_SECRET_ACCESS_KEY`
|
||||
- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_)
|
||||
|
||||
You can also define a custom ACL for exported feeds using this setting:
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
|
||||
|
||||
- :setting:`FEED_STORAGE_S3_ACL`
|
||||
- :setting:`AWS_ENDPOINT_URL`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
|
@ -208,19 +221,20 @@ Google Cloud Storage (GCS)
|
|||
|
||||
The feeds are stored on `Google Cloud Storage`_.
|
||||
|
||||
* URI scheme: ``gs``
|
||||
* Example URIs:
|
||||
- URI scheme: ``gs``
|
||||
|
||||
* ``gs://mybucket/path/to/export.csv``
|
||||
- Example URIs:
|
||||
|
||||
* Required external libraries: `google-cloud-storage`_.
|
||||
- ``gs://mybucket/path/to/export.csv``
|
||||
|
||||
- Required external libraries: `google-cloud-storage`_.
|
||||
|
||||
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
|
||||
|
||||
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
|
||||
|
||||
* :setting:`FEED_STORAGE_GCS_ACL`
|
||||
* :setting:`GCS_PROJECT_ID`
|
||||
- :setting:`FEED_STORAGE_GCS_ACL`
|
||||
- :setting:`GCS_PROJECT_ID`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
|
@ -234,9 +248,9 @@ Standard output
|
|||
|
||||
The feeds are written to the standard output of the Scrapy process.
|
||||
|
||||
* URI scheme: ``stdout``
|
||||
* Example URI: ``stdout:``
|
||||
* Required external libraries: none
|
||||
- URI scheme: ``stdout``
|
||||
- Example URI: ``stdout:``
|
||||
- Required external libraries: none
|
||||
|
||||
|
||||
.. _delayed-file-delivery:
|
||||
|
|
@ -259,21 +273,118 @@ soon as a file reaches the maximum item count, that file is delivered to the
|
|||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
|
||||
.. _item-filter:
|
||||
|
||||
Item filtering
|
||||
==============
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
You can filter items that you want to allow for a particular feed by using the
|
||||
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
|
||||
the specified types will be added to the feed.
|
||||
|
||||
The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
|
||||
class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
|
||||
|
||||
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
|
||||
method ``accepts`` and taking ``feed_options`` as an argument.
|
||||
|
||||
For instance::
|
||||
|
||||
class MyCustomFilter:
|
||||
|
||||
def __init__(self, feed_options):
|
||||
self.feed_options = feed_options
|
||||
|
||||
def accepts(self, item):
|
||||
if "field1" in item and item["field1"] == "expected_data":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
|
||||
See :setting:`FEEDS` for examples.
|
||||
|
||||
ItemFilter
|
||||
----------
|
||||
|
||||
.. autoclass:: scrapy.extensions.feedexport.ItemFilter
|
||||
:members:
|
||||
|
||||
|
||||
.. _post-processing:
|
||||
|
||||
Post-Processing
|
||||
===============
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
the feed to be processed. These plugins can be declared either as an import string
|
||||
or with the imported class of the plugin. Parameters to plugins can be passed
|
||||
through the feed options. See :ref:`feed options <feed-options>` for examples.
|
||||
|
||||
.. _builtin-plugins:
|
||||
|
||||
Built-in Plugins
|
||||
----------------
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
|
||||
|
||||
.. _custom-plugins:
|
||||
|
||||
Custom Plugins
|
||||
--------------
|
||||
|
||||
Each plugin is a class that must implement the following methods:
|
||||
|
||||
.. method:: __init__(self, file, feed_options)
|
||||
|
||||
Initialize the plugin.
|
||||
|
||||
:param file: file-like object having at least the `write`, `tell` and `close` methods implemented
|
||||
|
||||
:param feed_options: feed-specific :ref:`options <feed-options>`
|
||||
:type feed_options: :class:`dict`
|
||||
|
||||
.. method:: write(self, data)
|
||||
|
||||
Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
|
||||
It must return number of bytes written.
|
||||
|
||||
.. method:: close(self)
|
||||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
These are the settings used for configuring the feed exports:
|
||||
|
||||
* :setting:`FEEDS` (mandatory)
|
||||
* :setting:`FEED_EXPORT_ENCODING`
|
||||
* :setting:`FEED_STORE_EMPTY`
|
||||
* :setting:`FEED_EXPORT_FIELDS`
|
||||
* :setting:`FEED_EXPORT_INDENT`
|
||||
* :setting:`FEED_STORAGES`
|
||||
* :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
* :setting:`FEED_EXPORTERS`
|
||||
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
- :setting:`FEEDS` (mandatory)
|
||||
- :setting:`FEED_EXPORT_ENCODING`
|
||||
- :setting:`FEED_STORE_EMPTY`
|
||||
- :setting:`FEED_EXPORT_FIELDS`
|
||||
- :setting:`FEED_EXPORT_INDENT`
|
||||
- :setting:`FEED_STORAGES`
|
||||
- :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
- :setting:`FEED_STORAGE_S3_ACL`
|
||||
- :setting:`FEED_EXPORTERS`
|
||||
- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
|
||||
.. currentmodule:: scrapy.extensions.feedexport
|
||||
|
||||
|
|
@ -301,21 +412,26 @@ For instance::
|
|||
'format': 'json',
|
||||
'encoding': 'utf8',
|
||||
'store_empty': False,
|
||||
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
'export_empty_fields': True,
|
||||
},
|
||||
},
|
||||
},
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
'fields': ['name', 'price'],
|
||||
'item_filter': MyCustomFilter1,
|
||||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path('items.csv'): {
|
||||
pathlib.Path('items.csv.gz'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 5,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -337,6 +453,18 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
|
||||
|
||||
- ``item_classes``: list of :ref:`item classes <topics-items>` to export.
|
||||
|
||||
If undefined or empty, all items are exported.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
|
||||
|
||||
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
|
||||
|
|
@ -367,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
|
||||
|
||||
- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
|
||||
|
||||
The plugins will be used in the order of the list passed.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -600,9 +733,12 @@ The function signature should be as follows:
|
|||
:type params: dict
|
||||
|
||||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.spiders.Spider
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
#. Define the following function somewhere in your project::
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following
|
|||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
Additionally, they may also implement the following methods:
|
||||
|
||||
|
|
@ -51,18 +51,18 @@ Additionally, they may also implement the following methods:
|
|||
This method is called when the spider is opened.
|
||||
|
||||
:param spider: the spider which was opened
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: close_spider(self, spider)
|
||||
|
||||
This method is called when the spider is closed.
|
||||
|
||||
:param spider: the spider which was closed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a pipeline instance
|
||||
If present, this class method is called to create a pipeline instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the pipeline. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for pipeline to
|
||||
|
|
@ -190,6 +190,8 @@ item.
|
|||
|
||||
import scrapy
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
|
||||
class ScreenshotPipeline:
|
||||
"""Pipeline that uses Splash to render screenshot of
|
||||
|
|
@ -202,7 +204,7 @@ item.
|
|||
encoded_item_url = quote(adapter["url"])
|
||||
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
|
||||
request = scrapy.Request(screenshot_url)
|
||||
response = await spider.crawler.engine.download(request, spider)
|
||||
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
|
||||
|
||||
if response.status != 200:
|
||||
# Error happened, return item.
|
||||
|
|
|
|||
|
|
@ -42,7 +42,8 @@ Item objects
|
|||
:class:`Item` provides a :class:`dict`-like API plus additional features that
|
||||
make it the most feature-complete item type:
|
||||
|
||||
.. class:: Item([arg])
|
||||
.. class:: scrapy.item.Item([arg])
|
||||
.. class:: scrapy.Item([arg])
|
||||
|
||||
:class:`Item` objects replicate the standard :class:`dict` API, including
|
||||
its ``__init__`` method.
|
||||
|
|
@ -101,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
|
|||
* define custom field metadata through :func:`dataclasses.field`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
They work natively in Python 3.7 or later, or using the `dataclasses
|
||||
backport`_ in Python 3.6.
|
||||
|
||||
.. _dataclasses backport: https://pypi.org/project/dataclasses/
|
||||
|
||||
Example::
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -199,7 +195,8 @@ It's important to note that the :class:`Field` objects used to declare the item
|
|||
do not stay assigned as class attributes. Instead, they can be accessed through
|
||||
the :attr:`Item.fields` attribute.
|
||||
|
||||
.. class:: Field([arg])
|
||||
.. class:: scrapy.item.Field([arg])
|
||||
.. class:: scrapy.Field([arg])
|
||||
|
||||
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
|
||||
doesn't provide any extra functionality or attributes. In other words,
|
||||
|
|
@ -317,11 +314,11 @@ If that is not the desired behavior, use a deep copy instead.
|
|||
See :mod:`copy` for more information.
|
||||
|
||||
To create a shallow copy of an item, you can either call
|
||||
:meth:`~scrapy.item.Item.copy` on an existing item
|
||||
:meth:`~scrapy.Item.copy` on an existing item
|
||||
(``product2 = product.copy()``) or instantiate your item class from an existing
|
||||
item (``product2 = Product(product)``).
|
||||
|
||||
To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
|
||||
To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead
|
||||
(``product2 = product.deepcopy()``).
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
|
|||
|
||||
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
|
||||
|
||||
.. _topics-keeping-persistent-state-between-batches:
|
||||
|
||||
Keeping persistent state between batches
|
||||
========================================
|
||||
|
||||
|
|
@ -74,10 +76,10 @@ on cookies.
|
|||
Request serialization
|
||||
---------------------
|
||||
|
||||
For persistence to work, :class:`~scrapy.http.Request` objects must be
|
||||
For persistence to work, :class:`~scrapy.Request` objects must be
|
||||
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
|
||||
values passed to their ``__init__`` method, which must be methods of the
|
||||
running :class:`~scrapy.spiders.Spider` class.
|
||||
running :class:`~scrapy.Spider` class.
|
||||
|
||||
If you wish to log the requests that couldn't be serialized, you can set the
|
||||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ Common causes of memory leaks
|
|||
|
||||
It happens quite often (sometimes by accident, sometimes on purpose) that the
|
||||
Scrapy developer passes objects referenced in Requests (for example, using the
|
||||
:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
|
||||
:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta`
|
||||
attributes or the request callback function) and that effectively bounds the
|
||||
lifetime of those referenced objects to the lifetime of the Request. This is,
|
||||
by far, the most common cause of memory leaks in Scrapy projects, and a quite
|
||||
|
|
@ -48,9 +48,9 @@ Too Many Requests?
|
|||
------------------
|
||||
|
||||
By default Scrapy keeps the request queue in memory; it includes
|
||||
:class:`~scrapy.http.Request` objects and all objects
|
||||
referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
|
||||
and :attr:`~scrapy.http.Request.meta`).
|
||||
:class:`~scrapy.Request` objects and all objects
|
||||
referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs`
|
||||
and :attr:`~scrapy.Request.meta`).
|
||||
While not necessarily a leak, this can take a lot of memory. Enabling
|
||||
:ref:`persistent job queue <topics-jobs>` could help keeping memory usage
|
||||
in control.
|
||||
|
|
@ -90,11 +90,11 @@ Which objects are tracked?
|
|||
The objects tracked by ``trackrefs`` are all from these classes (and all its
|
||||
subclasses):
|
||||
|
||||
* :class:`scrapy.http.Request`
|
||||
* :class:`scrapy.Request`
|
||||
* :class:`scrapy.http.Response`
|
||||
* :class:`scrapy.item.Item`
|
||||
* :class:`scrapy.selector.Selector`
|
||||
* :class:`scrapy.spiders.Spider`
|
||||
* :class:`scrapy.Item`
|
||||
* :class:`scrapy.Selector`
|
||||
* :class:`scrapy.Spider`
|
||||
|
||||
A real example
|
||||
--------------
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ chapter <topics-items>`::
|
|||
l.add_xpath('name', '//div[@class="product_name"]')
|
||||
l.add_xpath('name', '//div[@class="product_title"]')
|
||||
l.add_xpath('price', '//p[@id="price"]')
|
||||
l.add_css('stock', 'p#stock]')
|
||||
l.add_css('stock', 'p#stock')
|
||||
l.add_value('last_updated', 'today') # you can also use literal values
|
||||
return l.load_item()
|
||||
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ path::
|
|||
Logging from Spiders
|
||||
====================
|
||||
|
||||
Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
|
||||
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
|
||||
instance, which can be accessed and used like this::
|
||||
|
||||
import scrapy
|
||||
|
|
@ -101,7 +101,7 @@ instance, which can be accessed and used like this::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
start_urls = ['https://scrapy.org']
|
||||
|
||||
def parse(self, response):
|
||||
self.logger.info('Parse function called on %s', response.url)
|
||||
|
|
@ -117,7 +117,7 @@ Python logger you want. For example::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
start_urls = ['https://scrapy.org']
|
||||
|
||||
def parse(self, response):
|
||||
logger.info('Parse function called on %s', response.url)
|
||||
|
|
@ -143,6 +143,7 @@ Logging settings
|
|||
These settings can be used to configure the logging:
|
||||
|
||||
* :setting:`LOG_FILE`
|
||||
* :setting:`LOG_FILE_APPEND`
|
||||
* :setting:`LOG_ENABLED`
|
||||
* :setting:`LOG_ENCODING`
|
||||
* :setting:`LOG_LEVEL`
|
||||
|
|
@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
|
|||
:setting:`LOG_FILE` is set, messages sent through the root logger will be
|
||||
redirected to a file named :setting:`LOG_FILE` with encoding
|
||||
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
|
||||
messages will be displayed on the standard error. Lastly, if
|
||||
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
|
||||
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
|
||||
(discarding the output from previous runs, if any). Lastly, if
|
||||
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
|
||||
|
||||
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
|
||||
|
|
@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many
|
|||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
|
|
@ -242,6 +245,47 @@ e.g. in the spider's ``__init__`` method::
|
|||
If you run this spider again then INFO messages from
|
||||
``scrapy.spidermiddlewares.httperror`` logger will be gone.
|
||||
|
||||
You can also filter log records by :class:`~logging.LogRecord` data. For
|
||||
example, you can filter log records by message content using a substring or
|
||||
a regular expression. Create a :class:`logging.Filter` subclass
|
||||
and equip it with a regular expression pattern to
|
||||
filter out unwanted messages::
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
class ContentFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
|
||||
if match:
|
||||
return False
|
||||
|
||||
A project-level filter may be attached to the root
|
||||
handler created by Scrapy, this is a wieldy way to
|
||||
filter all loggers in different parts of the project
|
||||
(middlewares, spider, etc.)::
|
||||
|
||||
import logging
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# ...
|
||||
def __init__(self, *args, **kwargs):
|
||||
for handler in logging.root.handlers:
|
||||
handler.addFilter(ContentFilter())
|
||||
|
||||
Alternatively, you may choose a specific logger
|
||||
and hide it without affecting other loggers::
|
||||
|
||||
import logging
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
# ...
|
||||
def __init__(self, *args, **kwargs):
|
||||
logger = logging.getLogger('my_logger')
|
||||
logger.addFilter(ContentFilter())
|
||||
|
||||
scrapy.utils.log module
|
||||
=======================
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
|||
can configure some extra functions like generating thumbnails and filtering
|
||||
the images based on their size.
|
||||
|
||||
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
|
||||
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
|
||||
thumbnailing and normalizing images to JPEG/RGB format.
|
||||
|
||||
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||
|
|
@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
|
|||
|
||||
IMAGES_STORE = '/path/to/valid/dir'
|
||||
|
||||
.. _topics-file-naming:
|
||||
|
||||
File Naming
|
||||
===========
|
||||
|
||||
Default File Naming
|
||||
-------------------
|
||||
|
||||
By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names.
|
||||
|
||||
For example, the following image URL::
|
||||
|
||||
http://www.example.com/image.jpg
|
||||
|
||||
Whose ``SHA-1 hash`` is::
|
||||
|
||||
3afec3b4765f8f0a07b78f98c07b83f013567a0a
|
||||
|
||||
Will be downloaded and stored using your chosen :ref:`storage method <topics-supported-storage>` and the following file name::
|
||||
|
||||
3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
|
||||
|
||||
Custom File Naming
|
||||
-------------------
|
||||
|
||||
You may wish to use a different calculated file name for saved files.
|
||||
For example, classifying an image by including meta in the file name.
|
||||
|
||||
Customize file names by overriding the ``file_path`` method of your
|
||||
media pipeline.
|
||||
|
||||
For example, an image pipeline with image URL::
|
||||
|
||||
http://www.example.com/product/images/large/front/0000000004166
|
||||
|
||||
Can be processed into a file name with a condensed hash and the perspective
|
||||
``front``::
|
||||
|
||||
00b08510e4_front.jpg
|
||||
|
||||
By overriding ``file_path`` like this:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import hashlib
|
||||
from os.path import splitext
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
|
||||
image_perspective = request.url.split('/')[-2]
|
||||
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
|
||||
|
||||
return image_filename
|
||||
|
||||
.. warning::
|
||||
If your custom file name scheme relies on meta data that can vary between
|
||||
scrapes it may lead to unexpected re-downloading of existing media using
|
||||
new file names.
|
||||
|
||||
For example, if your custom file name scheme uses a product title and the
|
||||
site changes an item's product title between scrapes, Scrapy will re-download
|
||||
the same media using updated file names.
|
||||
|
||||
For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`.
|
||||
|
||||
.. _topics-supported-storage:
|
||||
|
||||
Supported Storage
|
||||
=================
|
||||
|
||||
File system storage
|
||||
-------------------
|
||||
|
||||
The files are stored using a `SHA1 hash`_ of their URLs for the file names.
|
||||
File system storage will save files to the following path::
|
||||
|
||||
For example, the following image URL::
|
||||
|
||||
http://www.example.com/image.jpg
|
||||
|
||||
Whose ``SHA1 hash`` is::
|
||||
|
||||
3afec3b4765f8f0a07b78f98c07b83f013567a0a
|
||||
|
||||
Will be downloaded and stored in the following file::
|
||||
|
||||
<IMAGES_STORE>/full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
|
||||
<IMAGES_STORE>/full/<FILE_NAME>
|
||||
|
||||
Where:
|
||||
|
||||
|
|
@ -139,6 +196,9 @@ Where:
|
|||
* ``full`` is a sub-directory to separate full images from thumbnails (if
|
||||
used). For more info see :ref:`topics-images-thumbnails`.
|
||||
|
||||
* ``<FILE_NAME>`` is the file name assigned to the file. For more info see :ref:`topics-file-naming`.
|
||||
|
||||
|
||||
.. _media-pipeline-ftp:
|
||||
|
||||
FTP server storage
|
||||
|
|
@ -259,7 +319,7 @@ respectively), the pipeline will put the results under the respective field
|
|||
When using :ref:`item types <item-types>` for which fields are defined beforehand,
|
||||
you must define both the URLs field and the results field. For example, when
|
||||
using the images pipeline, items must define both the ``image_urls`` and the
|
||||
``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
|
||||
``images`` field. For instance, using the :class:`~scrapy.Item` class::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -296,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
|||
Additional features
|
||||
===================
|
||||
|
||||
.. _file-expiration:
|
||||
|
||||
File expiration
|
||||
---------------
|
||||
|
||||
|
|
@ -323,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
|
|||
|
||||
and pipeline class MyPipeline will have expiration time set to 180.
|
||||
|
||||
The last modified time from the file is used to determine the age of the file in days,
|
||||
which is then compared to the set expiration time to determine if the file is expired.
|
||||
|
||||
.. _topics-images-thumbnails:
|
||||
|
||||
Thumbnail generation for images
|
||||
|
|
@ -353,9 +418,9 @@ Where:
|
|||
* ``<size_name>`` is the one specified in the :setting:`IMAGES_THUMBS`
|
||||
dictionary keys (``small``, ``big``, etc)
|
||||
|
||||
* ``<image_id>`` is the `SHA1 hash`_ of the image url
|
||||
* ``<image_id>`` is the `SHA-1 hash`_ of the image url
|
||||
|
||||
.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
|
||||
.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
|
||||
|
||||
Example of image files stored using ``small`` and ``big`` thumbnail names::
|
||||
|
||||
|
|
@ -424,7 +489,7 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
:class:`item <scrapy.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -563,7 +628,7 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
:class:`item <scrapy.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -591,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
.. versionadded:: 2.4
|
||||
The *item* parameter.
|
||||
|
||||
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
|
||||
thumbnail download path of the image originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>`,
|
||||
``thumb_id``,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.Item>`.
|
||||
|
||||
You can override this method to customize the thumbnail download path of each image.
|
||||
You can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`thumb_path` method returns
|
||||
``thumbs/<size name>/<request URL hash>.<extension>``.
|
||||
|
||||
|
||||
.. method:: ImagesPipeline.get_media_requests(item, info)
|
||||
|
||||
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ project as example.
|
|||
process = CrawlerProcess(get_project_settings())
|
||||
|
||||
# 'followall' is the name of one of the spiders of the project.
|
||||
process.crawl('followall', domain='scrapinghub.com')
|
||||
process.crawl('followall', domain='scrapy.org')
|
||||
process.start() # the script will block here until the crawling is finished
|
||||
|
||||
There's another Scrapy utility that provides more control over the crawling
|
||||
|
|
@ -119,6 +119,7 @@ Here is an example that runs multiple spiders simultaneously:
|
|||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
class MySpider1(scrapy.Spider):
|
||||
# Your first spider definition
|
||||
|
|
@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously:
|
|||
# Your second spider definition
|
||||
...
|
||||
|
||||
process = CrawlerProcess()
|
||||
settings = get_project_settings()
|
||||
process = CrawlerProcess(settings)
|
||||
process.crawl(MySpider1)
|
||||
process.crawl(MySpider2)
|
||||
process.start() # the script will block here until all crawling jobs are finished
|
||||
|
|
@ -141,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
|
|||
from twisted.internet import reactor
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
class MySpider1(scrapy.Spider):
|
||||
# Your first spider definition
|
||||
|
|
@ -151,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
|
|||
...
|
||||
|
||||
configure_logging()
|
||||
runner = CrawlerRunner()
|
||||
settings = get_project_settings()
|
||||
runner = CrawlerRunner(settings)
|
||||
runner.crawl(MySpider1)
|
||||
runner.crawl(MySpider2)
|
||||
d = runner.join()
|
||||
|
|
@ -166,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
from twisted.internet import reactor, defer
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
from scrapy.utils.project import get_project_settings
|
||||
|
||||
class MySpider1(scrapy.Spider):
|
||||
# Your first spider definition
|
||||
|
|
@ -175,8 +180,9 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
# Your second spider definition
|
||||
...
|
||||
|
||||
configure_logging()
|
||||
runner = CrawlerRunner()
|
||||
settings = get_project_settings()
|
||||
configure_logging(settings)
|
||||
runner = CrawlerRunner(settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def crawl():
|
||||
|
|
@ -187,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
crawl()
|
||||
reactor.run() # the script will block here until the last crawl call is finished
|
||||
|
||||
Different spiders can set different values for the same setting, but when they
|
||||
run in the same process it may be impossible, by design or because of some
|
||||
limitations, to use these different values. What happens in practice is
|
||||
different for different settings:
|
||||
|
||||
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
|
||||
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
|
||||
default one) cannot be read from the per-spider settings. These are applied
|
||||
when the :class:`~scrapy.crawler.CrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerProcess` object is created.
|
||||
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
|
||||
available value is used, and if a spider requests a different reactor an
|
||||
exception will be raised. These are applied when the reactor is installed.
|
||||
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
|
||||
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
|
||||
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
|
||||
the first available value is used. These are applied when the reactor is
|
||||
started.
|
||||
|
||||
.. seealso:: :ref:`run-from-script`.
|
||||
|
||||
.. _distributed-crawls:
|
||||
|
|
@ -237,14 +262,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
|
||||
cookies to spot bot behaviour
|
||||
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
|
||||
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
|
||||
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
||||
directly
|
||||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
super proxy that you can attach your own proxies to.
|
||||
* use a highly distributed downloader that circumvents bans internally, so you
|
||||
can just focus on parsing clean pages. One example of such downloaders is
|
||||
`Crawlera`_
|
||||
`Zyte Smart Proxy Manager`_
|
||||
|
||||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
|
@ -252,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
|
|
|||
|
|
@ -26,10 +26,6 @@ Request objects
|
|||
|
||||
.. autoclass:: Request
|
||||
|
||||
A :class:`Request` object represents an HTTP request, which is usually
|
||||
generated in the Spider and executed by the Downloader, and thus generating
|
||||
a :class:`Response`.
|
||||
|
||||
:param url: the URL of this request
|
||||
|
||||
If the URL is invalid, a :exc:`ValueError` exception is raised.
|
||||
|
|
@ -39,7 +35,7 @@ Request objects
|
|||
request (once it's downloaded) as its first parameter. For more information
|
||||
see :ref:`topics-request-response-ref-request-callback-arguments` below.
|
||||
If a Request doesn't specify a callback, the spider's
|
||||
:meth:`~scrapy.spiders.Spider.parse` method will be used.
|
||||
:meth:`~scrapy.Spider.parse` method will be used.
|
||||
Note that if exceptions are raised during processing, errback is called instead.
|
||||
|
||||
:type callback: collections.abc.Callable
|
||||
|
|
@ -64,7 +60,7 @@ Request objects
|
|||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
:type headers: dict
|
||||
|
|
@ -96,7 +92,7 @@ Request objects
|
|||
|
||||
To create a request that does not send stored cookies and does not
|
||||
store received cookies, set the ``dont_merge_cookies`` key to ``True``
|
||||
in :attr:`request.meta <scrapy.http.Request.meta>`.
|
||||
in :attr:`request.meta <scrapy.Request.meta>`.
|
||||
|
||||
Example of a request that sends manually-defined cookies and ignores
|
||||
cookie storage::
|
||||
|
|
@ -111,9 +107,13 @@ Request objects
|
|||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
|
||||
are casted to :class:`str`.
|
||||
|
||||
:type cookies: dict or list
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
@ -205,6 +205,8 @@ Request objects
|
|||
``failure.request.cb_kwargs`` in the request's errback. For more information,
|
||||
see :ref:`errback-cb_kwargs`.
|
||||
|
||||
.. autoattribute:: Request.attributes
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
Return a new Request which is a copy of this Request. See also:
|
||||
|
|
@ -220,6 +222,15 @@ Request objects
|
|||
|
||||
.. automethod:: from_curl
|
||||
|
||||
.. automethod:: to_dict
|
||||
|
||||
|
||||
Other functions related to requests
|
||||
-----------------------------------
|
||||
|
||||
.. autofunction:: scrapy.utils.request.request_from_dict
|
||||
|
||||
|
||||
.. _topics-request-response-ref-request-callback-arguments:
|
||||
|
||||
Passing additional data to callback functions
|
||||
|
|
@ -293,7 +304,7 @@ errors if needed::
|
|||
"http://www.httpbin.org/status/404", # Not found error
|
||||
"http://www.httpbin.org/status/500", # server issue
|
||||
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
|
||||
"http://www.httphttpbinbin.org/", # DNS error expected
|
||||
"https://example.invalid/", # DNS error expected
|
||||
]
|
||||
|
||||
def start_requests(self):
|
||||
|
|
@ -328,6 +339,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
|
|
@ -353,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
|
|||
main_url=failure.request.cb_kwargs['main_url'],
|
||||
)
|
||||
|
||||
|
||||
.. _request-fingerprints:
|
||||
|
||||
Request fingerprints
|
||||
--------------------
|
||||
|
||||
There are some aspects of scraping, such as filtering out duplicate requests
|
||||
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
|
||||
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
|
||||
unique identifier from a :class:`~scrapy.http.Request` object: a request
|
||||
fingerprint.
|
||||
|
||||
You often do not need to worry about request fingerprints, the default request
|
||||
fingerprinter works for most projects.
|
||||
|
||||
However, there is no universal way to generate a unique identifier from a
|
||||
request, because different situations require comparing requests differently.
|
||||
For example, sometimes you may need to compare URLs case-insensitively, include
|
||||
URL fragments, exclude certain URL query parameters, include some or all
|
||||
headers, etc.
|
||||
|
||||
To change how request fingerprints are built for your requests, use the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_CLASS
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: :class:`scrapy.utils.request.RequestFingerprinter`
|
||||
|
||||
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
|
||||
import path.
|
||||
|
||||
.. autoclass:: scrapy.utils.request.RequestFingerprinter
|
||||
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: ``'PREVIOUS_VERSION'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'PREVIOUS_VERSION'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy PREVIOUS_VERSION and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'VERSION'``
|
||||
|
||||
This implementation was introduced in Scrapy VERSION to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidade the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'PREVIOUS_VERSION'``).
|
||||
|
||||
|
||||
.. _PREVIOUS_VERSION-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A request fingerprinter is a class that must implement the following method:
|
||||
|
||||
.. method:: fingerprint(self, request)
|
||||
|
||||
Return a :class:`bytes` object that uniquely identifies *request*.
|
||||
|
||||
See also :ref:`request-fingerprint-restrictions`.
|
||||
|
||||
:param request: request to fingerprint
|
||||
:type request: scrapy.http.Request
|
||||
|
||||
Additionally, it may also implement the following methods:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this class method is called to create a request fingerprinter
|
||||
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
|
||||
new instance of the request fingerprinter.
|
||||
|
||||
*crawler* provides access to all Scrapy core components like settings and
|
||||
signals; it is a way for the request fingerprinter to access them and hook
|
||||
its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this request fingerprinter
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
.. classmethod:: from_settings(cls, settings)
|
||||
|
||||
If present, and ``from_crawler`` is not defined, this class method is called
|
||||
to create a request fingerprinter instance from a
|
||||
:class:`~scrapy.settings.Settings` object. It must return a new instance of
|
||||
the request fingerprinter.
|
||||
|
||||
The ``fingerprint`` method of the default request fingerprinter,
|
||||
:class:`scrapy.utils.request.RequestFingerprinter`, uses
|
||||
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
|
||||
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
|
||||
in your ``fingerprint`` method implementation:
|
||||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
For example, to take the value of a request header named ``X-ID`` into
|
||||
account::
|
||||
|
||||
# my_project/settings.py
|
||||
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
|
||||
|
||||
# my_project/utils.py
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
You can also write your own fingerprinting logic from scratch.
|
||||
|
||||
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
|
||||
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
|
||||
|
||||
- Caching saves CPU by ensuring that fingerprints are calculated only once
|
||||
per request, and not once per Scrapy component that needs the fingerprint
|
||||
of a request.
|
||||
|
||||
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
|
||||
request objects do not stay in memory forever just because you have
|
||||
references to them in your cache dictionary.
|
||||
|
||||
For example, to take into account only the URL of a request, without any prior
|
||||
URL canonicalization or taking the request method or body into account::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
If you need to be able to override the request fingerprinting for arbitrary
|
||||
requests from your spider callbacks, you may implement a request fingerprinter
|
||||
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
|
||||
when available, and then falls back to
|
||||
:func:`~scrapy.utils.request.fingerprint`. For example::
|
||||
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
|
||||
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
|
||||
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
|
||||
request fingerprinter::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
|
||||
.. _request-fingerprint-restrictions:
|
||||
|
||||
Request fingerprint restrictions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapy components that use request fingerprints may impose additional
|
||||
restrictions on the format of the fingerprints that your :ref:`request
|
||||
fingerprinter <custom-request-fingerprinter>` generates.
|
||||
|
||||
The following built-in Scrapy components have such restrictions:
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
|
||||
value of :setting:`HTTPCACHE_STORAGE`)
|
||||
|
||||
Request fingerprints must be at least 1 byte long.
|
||||
|
||||
Path and filename length limits of the file system of
|
||||
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
|
||||
the following directory structure is created:
|
||||
|
||||
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
|
||||
|
||||
- first byte of a request fingerprint as hexadecimal
|
||||
|
||||
- fingerprint as hexadecimal
|
||||
|
||||
- filenames up to 16 characters long
|
||||
|
||||
For example, if a request fingerprint is made of 20 bytes (default),
|
||||
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
|
||||
and the name of your spider is ``'my_spider'`` your file system must
|
||||
support a file path like::
|
||||
|
||||
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
|
||||
|
||||
The underlying DBM implementation must support keys as long as twice
|
||||
the number of bytes of a request fingerprint, plus 5. For example,
|
||||
if a request fingerprint is made of 20 bytes (default),
|
||||
45-character-long keys must be supported.
|
||||
|
||||
|
||||
.. _topics-request-meta:
|
||||
|
||||
Request.meta special keys
|
||||
|
|
@ -363,26 +642,26 @@ are some special keys recognized by Scrapy and its built-in extensions.
|
|||
|
||||
Those are:
|
||||
|
||||
* :reqmeta:`dont_redirect`
|
||||
* :reqmeta:`dont_retry`
|
||||
* :reqmeta:`handle_httpstatus_list`
|
||||
* :reqmeta:`handle_httpstatus_all`
|
||||
* :reqmeta:`dont_merge_cookies`
|
||||
* :reqmeta:`bindaddress`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
* :reqmeta:`dont_merge_cookies`
|
||||
* :reqmeta:`dont_obey_robotstxt`
|
||||
* :reqmeta:`dont_redirect`
|
||||
* :reqmeta:`dont_retry`
|
||||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_timeout`
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
|
||||
* :reqmeta:`handle_httpstatus_all`
|
||||
* :reqmeta:`handle_httpstatus_list`
|
||||
* :reqmeta:`max_retry_times`
|
||||
* :reqmeta:`proxy`
|
||||
* :reqmeta:`redirect_reasons`
|
||||
* :reqmeta:`redirect_urls`
|
||||
* :reqmeta:`bindaddress`
|
||||
* :reqmeta:`dont_obey_robotstxt`
|
||||
* :reqmeta:`download_timeout`
|
||||
* :reqmeta:`download_maxsize`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`proxy`
|
||||
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
* :reqmeta:`referrer_policy`
|
||||
* :reqmeta:`max_retry_times`
|
||||
|
||||
.. reqmeta:: bindaddress
|
||||
|
||||
|
|
@ -432,9 +711,9 @@ The meta key is used set retry times per request. When initialized, the
|
|||
Stopping the download of a Response
|
||||
===================================
|
||||
|
||||
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
|
||||
:class:`~scrapy.signals.bytes_received` signal handler will stop the
|
||||
download of a given response. See the following example::
|
||||
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
|
||||
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
|
||||
signals will stop the download of a given response. See the following example::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -488,7 +767,9 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
|
||||
|
||||
.. class:: FormRequest(url, [formdata, ...])
|
||||
.. class:: scrapy.http.request.form.FormRequest
|
||||
.. class:: scrapy.http.FormRequest
|
||||
.. class:: scrapy.FormRequest(url, [formdata, ...])
|
||||
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
|
|
@ -642,6 +923,8 @@ dealing with JSON requests.
|
|||
data into JSON format.
|
||||
:type dumps_kwargs: dict
|
||||
|
||||
.. autoattribute:: JsonRequest.attributes
|
||||
|
||||
JsonRequest usage example
|
||||
-------------------------
|
||||
|
||||
|
|
@ -659,9 +942,6 @@ Response objects
|
|||
|
||||
.. autoclass:: Response
|
||||
|
||||
A :class:`Response` object represents an HTTP response, which is usually
|
||||
downloaded (by the Downloader) and fed to the Spiders for processing.
|
||||
|
||||
:param url: the URL of this response
|
||||
:type url: str
|
||||
|
||||
|
|
@ -685,7 +965,7 @@ Response objects
|
|||
|
||||
:param request: the initial value of the :attr:`Response.request` attribute.
|
||||
This represents the :class:`Request` that generated this response.
|
||||
:type request: scrapy.http.Request
|
||||
:type request: scrapy.Request
|
||||
|
||||
:param certificate: an object representing the server's SSL certificate.
|
||||
:type certificate: twisted.internet.ssl.Certificate
|
||||
|
|
@ -693,9 +973,19 @@ Response objects
|
|||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
||||
:param protocol: The protocol that was used to download the response.
|
||||
For instance: "HTTP/1.0", "HTTP/1.1", "h2"
|
||||
:type protocol: :class:`str`
|
||||
|
||||
.. versionadded:: 2.0.0
|
||||
The ``certificate`` parameter.
|
||||
|
||||
.. versionadded:: 2.1.0
|
||||
The ``ip_address`` parameter.
|
||||
|
||||
.. versionadded:: 2.5.0
|
||||
The ``protocol`` parameter.
|
||||
|
||||
.. attribute:: Response.url
|
||||
|
||||
A string containing the URL of the response.
|
||||
|
|
@ -780,6 +1070,8 @@ Response objects
|
|||
|
||||
.. attribute:: Response.certificate
|
||||
|
||||
.. versionadded:: 2.0.0
|
||||
|
||||
A :class:`twisted.internet.ssl.Certificate` object representing
|
||||
the server's SSL certificate.
|
||||
|
||||
|
|
@ -795,6 +1087,19 @@ Response objects
|
|||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. attribute:: Response.protocol
|
||||
|
||||
.. versionadded:: 2.5.0
|
||||
|
||||
The protocol that was used to download the response.
|
||||
For instance: "HTTP/1.0", "HTTP/1.1"
|
||||
|
||||
This attribute is currently only populated by the HTTP download
|
||||
handlers, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`protocol` is always ``None``.
|
||||
|
||||
.. autoattribute:: Response.attributes
|
||||
|
||||
.. method:: Response.copy()
|
||||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
|
@ -888,9 +1193,11 @@ TextResponse objects
|
|||
|
||||
.. attribute:: TextResponse.selector
|
||||
|
||||
A :class:`~scrapy.selector.Selector` instance using the response as
|
||||
A :class:`~scrapy.Selector` instance using the response as
|
||||
target. The selector is lazily instantiated on first access.
|
||||
|
||||
.. autoattribute:: TextResponse.attributes
|
||||
|
||||
:class:`TextResponse` objects support the following methods in addition to
|
||||
the standard :class:`Response` ones:
|
||||
|
||||
|
|
@ -915,6 +1222,14 @@ TextResponse objects
|
|||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
.. method:: TextResponse.urljoin(url)
|
||||
|
||||
Constructs an absolute url by combining the Response's base url with
|
||||
a possible relative url. The base url shall be extracted from the
|
||||
``<base>`` tag, or just the Response's :attr:`url` if there is no such
|
||||
tag.
|
||||
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
--------------------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
.. _topics-scheduler:
|
||||
|
||||
=========
|
||||
Scheduler
|
||||
=========
|
||||
|
||||
.. module:: scrapy.core.scheduler
|
||||
|
||||
The scheduler component receives requests from the :ref:`engine <component-engine>`
|
||||
and stores them into persistent and/or non-persistent data structures.
|
||||
It also gets those requests and feeds them back to the engine when it
|
||||
asks for a next request to be downloaded.
|
||||
|
||||
|
||||
Overriding the default scheduler
|
||||
================================
|
||||
|
||||
You can use your own custom scheduler class by supplying its full
|
||||
Python path in the :setting:`SCHEDULER` setting.
|
||||
|
||||
|
||||
Minimal scheduler interface
|
||||
===========================
|
||||
|
||||
.. autoclass:: BaseScheduler
|
||||
:members:
|
||||
|
||||
|
||||
Default Scrapy scheduler
|
||||
========================
|
||||
|
||||
.. autoclass:: Scheduler
|
||||
:members:
|
||||
:special-members: __len__
|
||||
|
|
@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
|
|||
to extract data from the HTML source. There are several libraries available to
|
||||
achieve this, such as:
|
||||
|
||||
* `BeautifulSoup`_ is a very popular web scraping library among Python
|
||||
programmers which constructs a Python object based on the structure of the
|
||||
HTML code and also deals with bad markup reasonably well, but it has one
|
||||
drawback: it's slow.
|
||||
- `BeautifulSoup`_ is a very popular web scraping library among Python
|
||||
programmers which constructs a Python object based on the structure of the
|
||||
HTML code and also deals with bad markup reasonably well, but it has one
|
||||
drawback: it's slow.
|
||||
|
||||
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
|
||||
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
|
||||
library.)
|
||||
- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
|
||||
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
|
||||
standard library.)
|
||||
|
||||
Scrapy comes with its own mechanism for extracting data. They're called
|
||||
selectors because they "select" certain parts of the HTML document specified
|
||||
|
|
@ -48,7 +48,7 @@ Constructing selectors
|
|||
|
||||
.. highlight:: python
|
||||
|
||||
Response objects expose a :class:`~scrapy.selector.Selector` instance
|
||||
Response objects expose a :class:`~scrapy.Selector` instance
|
||||
on ``.selector`` attribute:
|
||||
|
||||
>>> response.selector.xpath('//span/text()').get()
|
||||
|
|
@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
|
|||
>>> response.css('span::text').get()
|
||||
'good'
|
||||
|
||||
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
|
||||
Scrapy selectors are instances of :class:`~scrapy.Selector` class
|
||||
constructed by passing either :class:`~scrapy.http.TextResponse` object or
|
||||
markup as a string (in ``text`` argument).
|
||||
|
||||
|
|
@ -175,7 +175,7 @@ of ``None``:
|
|||
'not-found'
|
||||
|
||||
Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
|
||||
using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
|
||||
using ``.attrib`` property of a :class:`~scrapy.Selector`:
|
||||
|
||||
>>> [img.attrib['src'] for img in response.css('img')]
|
||||
['image1_thumb.jpg',
|
||||
|
|
@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page):
|
|||
Using selectors with regular expressions
|
||||
----------------------------------------
|
||||
|
||||
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
|
||||
:class:`~scrapy.Selector` also has a ``.re()`` method for extracting
|
||||
data using regular expressions. However, unlike using ``.xpath()`` or
|
||||
``.css()`` methods, ``.re()`` returns a list of strings. So you
|
||||
can't construct nested ``.re()`` calls.
|
||||
|
|
@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
|
|||
you may want to take a look first at this `XPath tutorial`_.
|
||||
|
||||
.. note::
|
||||
Some of the tips are based on `this post from ScrapingHub's blog`_.
|
||||
Some of the tips are based on `this post from Zyte's blog`_.
|
||||
|
||||
.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
|
||||
.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
|
||||
.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
|
||||
|
||||
|
||||
.. _topics-selectors-relative-xpaths:
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ Example::
|
|||
|
||||
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
|
||||
own settings that will take precedence and override the project ones. They can
|
||||
do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute::
|
||||
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
|
@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``::
|
|||
The ``settings`` attribute is set in the base Spider class after the spider
|
||||
is initialized. If you want to use the settings before the initialization
|
||||
(e.g., in your spider's ``__init__()`` method), you'll need to override the
|
||||
:meth:`~scrapy.spiders.Spider.from_crawler` method.
|
||||
:meth:`~scrapy.Spider.from_crawler` method.
|
||||
|
||||
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
|
||||
attribute of the Crawler that is passed to ``from_crawler`` method in
|
||||
|
|
@ -204,6 +204,19 @@ Default: ``None``
|
|||
The AWS secret key used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: AWS_SESSION_TOKEN
|
||||
|
||||
AWS_SESSION_TOKEN
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The AWS security token used by code that requires access to `Amazon Web services`_,
|
||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
||||
`temporary security credentials`_.
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
|
|
@ -338,7 +351,7 @@ is non-zero, download delay is enforced per IP, not per domain.
|
|||
DEFAULT_ITEM_CLASS
|
||||
------------------
|
||||
|
||||
Default: ``'scrapy.item.Item'``
|
||||
Default: ``'scrapy.Item'``
|
||||
|
||||
The default class that will be used for instantiating items in the :ref:`the
|
||||
Scrapy shell <topics-shell>`.
|
||||
|
|
@ -360,7 +373,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the
|
|||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. setting:: DEPTH_LIMIT
|
||||
|
|
@ -384,8 +397,8 @@ Default: ``0``
|
|||
|
||||
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
|
||||
|
||||
An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
|
||||
a :class:`~scrapy.http.Request` based on its depth.
|
||||
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
|
||||
a :class:`~scrapy.Request` based on its depth.
|
||||
|
||||
The priority of a request is adjusted as follows::
|
||||
|
||||
|
|
@ -657,6 +670,7 @@ DOWNLOAD_HANDLERS_BASE
|
|||
Default::
|
||||
|
||||
{
|
||||
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
|
||||
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
|
||||
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
|
||||
|
|
@ -677,6 +691,45 @@ handler (without replacement), place this in your ``settings.py``::
|
|||
'ftp': None,
|
||||
}
|
||||
|
||||
.. _http2:
|
||||
|
||||
The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
|
||||
|
||||
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
|
||||
enable HTTP/2 support in Twisted.
|
||||
|
||||
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
|
||||
|
||||
DOWNLOAD_HANDLERS = {
|
||||
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
|
||||
}
|
||||
|
||||
.. warning::
|
||||
|
||||
HTTP/2 support in Scrapy is experimental, and not yet recommended for
|
||||
production environments. Future Scrapy versions may introduce related
|
||||
changes without a deprecation period or warning.
|
||||
|
||||
.. note::
|
||||
|
||||
Known limitations of the current HTTP/2 implementation of Scrapy include:
|
||||
|
||||
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
|
||||
HTTP/2 unencrypted (refer `http2 faq`_).
|
||||
|
||||
- No setting to specify a maximum `frame size`_ larger than the default
|
||||
value, 16384. Connections to servers that send a larger frame will
|
||||
fail.
|
||||
|
||||
- No support for `server pushes`_, which are ignored.
|
||||
|
||||
- No support for the :signal:`bytes_received` and
|
||||
:signal:`headers_received` signals.
|
||||
|
||||
.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
|
||||
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
|
||||
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
|
||||
|
||||
.. setting:: DOWNLOAD_TIMEOUT
|
||||
|
||||
DOWNLOAD_TIMEOUT
|
||||
|
|
@ -754,6 +807,15 @@ Optionally, this can be set per-request basis by using the
|
|||
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
|
||||
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
|
||||
|
||||
.. warning::
|
||||
|
||||
This setting is ignored by the
|
||||
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
|
||||
download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
|
||||
error, the corresponding HTTP/2 connection may be corrupted, affecting other
|
||||
requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
|
||||
failure is always raised for every request that was using that connection.
|
||||
|
||||
.. setting:: DUPEFILTER_CLASS
|
||||
|
||||
DUPEFILTER_CLASS
|
||||
|
|
@ -763,18 +825,14 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
|
|||
|
||||
The class used to detect and filter duplicate requests.
|
||||
|
||||
The default (``RFPDupeFilter``) filters based on request fingerprint using
|
||||
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
|
||||
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
|
||||
override its ``request_fingerprint`` method. This method should accept
|
||||
scrapy :class:`~scrapy.http.Request` object and return its fingerprint
|
||||
(a string).
|
||||
The default (``RFPDupeFilter``) filters based on the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
You can disable filtering of duplicate requests by setting
|
||||
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
|
||||
Be very careful about this however, because you can get into crawling loops.
|
||||
It's usually a better idea to set the ``dont_filter`` parameter to
|
||||
``True`` on the specific :class:`~scrapy.http.Request` that should not be
|
||||
``True`` on the specific :class:`~scrapy.Request` that should not be
|
||||
filtered.
|
||||
|
||||
.. setting:: DUPEFILTER_DEBUG
|
||||
|
|
@ -927,6 +985,16 @@ Default: ``{}``
|
|||
A dict containing the pipelines enabled by default in Scrapy. You should never
|
||||
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
|
||||
|
||||
.. setting:: JOBDIR
|
||||
|
||||
JOBDIR
|
||||
------
|
||||
|
||||
Default: ``''``
|
||||
|
||||
A string indicating the directory for storing the state of a crawl when
|
||||
:ref:`pausing and resuming crawls <topics-jobs>`.
|
||||
|
||||
.. setting:: LOG_ENABLED
|
||||
|
||||
LOG_ENABLED
|
||||
|
|
@ -954,6 +1022,16 @@ Default: ``None``
|
|||
|
||||
File name to use for logging output. If ``None``, standard error will be used.
|
||||
|
||||
.. setting:: LOG_FILE_APPEND
|
||||
|
||||
LOG_FILE_APPEND
|
||||
---------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
If ``False``, the log file specified with :setting:`LOG_FILE` will be
|
||||
overwritten (discarding the output from previous runs, if any).
|
||||
|
||||
.. setting:: LOG_FORMAT
|
||||
|
||||
LOG_FORMAT
|
||||
|
|
@ -1188,20 +1266,6 @@ Adjust redirect request priority relative to original request:
|
|||
- **a positive priority adjust (default) means higher priority.**
|
||||
- a negative priority adjust means lower priority.
|
||||
|
||||
.. setting:: RETRY_PRIORITY_ADJUST
|
||||
|
||||
RETRY_PRIORITY_ADJUST
|
||||
---------------------
|
||||
|
||||
Default: ``-1``
|
||||
|
||||
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
|
||||
|
||||
Adjust retry request priority relative to original request:
|
||||
|
||||
- a positive priority adjust means higher priority.
|
||||
- **a negative priority adjust (default) means lower priority.**
|
||||
|
||||
.. setting:: ROBOTSTXT_OBEY
|
||||
|
||||
ROBOTSTXT_OBEY
|
||||
|
|
@ -1249,7 +1313,8 @@ SCHEDULER
|
|||
|
||||
Default: ``'scrapy.core.scheduler.Scheduler'``
|
||||
|
||||
The scheduler to use for crawling.
|
||||
The scheduler class to be used for crawling.
|
||||
See the :ref:`topics-scheduler` topic for details.
|
||||
|
||||
.. setting:: SCHEDULER_DEBUG
|
||||
|
||||
|
|
@ -1507,7 +1572,7 @@ If a reactor is already installed,
|
|||
|
||||
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
|
||||
:exc:`Exception` if the installed reactor does not match the
|
||||
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
|
||||
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
|
||||
:mod:`~twisted.internet.reactor` imports in project files and imported
|
||||
third-party libraries will make Scrapy raise :exc:`Exception` when
|
||||
it checks which reactor is installed.
|
||||
|
|
@ -1528,7 +1593,7 @@ In order to use the reactor installed by Scrapy::
|
|||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1556,7 +1621,7 @@ which raises :exc:`Exception`, becomes::
|
|||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1569,10 +1634,9 @@ which raises :exc:`Exception`, becomes::
|
|||
|
||||
|
||||
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
|
||||
means that Scrapy will not attempt to install any specific reactor, and the
|
||||
default reactor defined by Twisted for the current platform will be used. This
|
||||
is to maintain backward compatibility and avoid possible problems caused by
|
||||
using a non-default reactor.
|
||||
means that Scrapy will install the default reactor defined by Twisted for the
|
||||
current platform. This is to maintain backward compatibility and avoid possible
|
||||
problems caused by using a non-default reactor.
|
||||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
|
|
@ -1586,8 +1650,19 @@ Default: ``2083``
|
|||
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs. For more information about
|
||||
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
This setting can act as a stopping condition in case of URLs of ever-increasing
|
||||
length, which may be caused for example by a programming error either in the
|
||||
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
|
||||
:setting:`DEPTH_LIMIT`.
|
||||
|
||||
Use ``0`` to allow URLs of any length.
|
||||
|
||||
The default value is copied from the `Microsoft Internet Explorer maximum URL
|
||||
length`_, even though this setting exists for different reasons.
|
||||
|
||||
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
|
||||
.. setting:: USER_AGENT
|
||||
|
||||
|
|
@ -1599,7 +1674,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
|
|||
The default User-Agent to use when crawling, unless overridden. This user agent is
|
||||
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
|
||||
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
|
||||
there is no overridding User-Agent header specified for the request.
|
||||
there is no overriding User-Agent header specified for the request.
|
||||
|
||||
|
||||
Settings documented elsewhere:
|
||||
|
|
@ -1610,7 +1685,6 @@ case to see how to enable and use them.
|
|||
|
||||
.. settingslist::
|
||||
|
||||
|
||||
.. _Amazon web services: https://aws.amazon.com/
|
||||
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
|
||||
|
|
|
|||
|
|
@ -95,20 +95,21 @@ convenience.
|
|||
Available Shortcuts
|
||||
-------------------
|
||||
|
||||
* ``shelp()`` - print a help with the list of available objects and shortcuts
|
||||
- ``shelp()`` - print a help with the list of available objects and
|
||||
shortcuts
|
||||
|
||||
* ``fetch(url[, redirect=True])`` - fetch a new response from the given
|
||||
URL and update all related objects accordingly. You can optionaly ask for
|
||||
HTTP 3xx redirections to not be followed by passing ``redirect=False``
|
||||
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
|
||||
and update all related objects accordingly. You can optionally ask for HTTP
|
||||
3xx redirections to not be followed by passing ``redirect=False``
|
||||
|
||||
* ``fetch(request)`` - fetch a new response from the given request and
|
||||
update all related objects accordingly.
|
||||
- ``fetch(request)`` - fetch a new response from the given request and update
|
||||
all related objects accordingly.
|
||||
|
||||
* ``view(response)`` - open the given response in your local web browser, for
|
||||
inspection. This will add a `\<base\> tag`_ to the response body in order
|
||||
for external links (such as images and style sheets) to display properly.
|
||||
Note, however, that this will create a temporary file in your computer,
|
||||
which won't be removed automatically.
|
||||
- ``view(response)`` - open the given response in your local web browser, for
|
||||
inspection. This will add a `\<base\> tag`_ to the response body in order
|
||||
for external links (such as images and style sheets) to display properly.
|
||||
Note, however, that this will create a temporary file in your computer,
|
||||
which won't be removed automatically.
|
||||
|
||||
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
|
||||
|
||||
|
|
@ -117,26 +118,26 @@ Available Scrapy objects
|
|||
|
||||
The Scrapy shell automatically creates some convenient objects from the
|
||||
downloaded page, like the :class:`~scrapy.http.Response` object and the
|
||||
:class:`~scrapy.selector.Selector` objects (for both HTML and XML
|
||||
:class:`~scrapy.Selector` objects (for both HTML and XML
|
||||
content).
|
||||
|
||||
Those objects are:
|
||||
|
||||
* ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
|
||||
- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
|
||||
|
||||
* ``spider`` - the Spider which is known to handle the URL, or a
|
||||
:class:`~scrapy.spiders.Spider` object if there is no spider found for
|
||||
the current URL
|
||||
- ``spider`` - the Spider which is known to handle the URL, or a
|
||||
:class:`~scrapy.Spider` object if there is no spider found for the
|
||||
current URL
|
||||
|
||||
* ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
|
||||
page. You can modify this request using :meth:`~scrapy.http.Request.replace`
|
||||
or fetch a new request (without leaving the shell) using the ``fetch``
|
||||
shortcut.
|
||||
- ``request`` - a :class:`~scrapy.Request` object of the last fetched
|
||||
page. You can modify this request using
|
||||
:meth:`~scrapy.Request.replace` or fetch a new request (without
|
||||
leaving the shell) using the ``fetch`` shortcut.
|
||||
|
||||
* ``response`` - a :class:`~scrapy.http.Response` object containing the last
|
||||
fetched page
|
||||
- ``response`` - a :class:`~scrapy.http.Response` object containing the last
|
||||
fetched page
|
||||
|
||||
* ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
|
||||
- ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
|
||||
|
||||
Example of shell session
|
||||
========================
|
||||
|
|
|
|||
|
|
@ -51,16 +51,16 @@ Deferred signal handlers
|
|||
========================
|
||||
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, allowing you to run asynchronous code that
|
||||
does not block Scrapy. If a signal handler returns a
|
||||
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
|
||||
:class:`~twisted.internet.defer.Deferred` to fire.
|
||||
or :term:`awaitable objects <awaitable>` from their handlers, allowing
|
||||
you to run asynchronous code that does not block Scrapy. If a signal
|
||||
handler returns one of these objects, Scrapy waits for that asynchronous
|
||||
operation to finish.
|
||||
|
||||
Let's take an example::
|
||||
Let's take an example using :ref:`coroutines <topics-coroutines>`::
|
||||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
start_urls = ['https://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
|
|
@ -68,17 +68,15 @@ Let's take an example::
|
|||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
def item_scraped(self, item):
|
||||
async def item_scraped(self, item):
|
||||
# Send the scraped item to the server
|
||||
d = treq.post(
|
||||
response = await treq.post(
|
||||
'http://example.com/post',
|
||||
json.dumps(item).encode('ascii'),
|
||||
headers={b'Content-Type': [b'application/json']}
|
||||
)
|
||||
|
||||
# The next item will be scraped only after
|
||||
# deferred (d) is fired
|
||||
return d
|
||||
return response
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
|
|
@ -89,7 +87,7 @@ Let's take an example::
|
|||
}
|
||||
|
||||
See the :ref:`topics-signals-ref` below to know which signals support
|
||||
:class:`~twisted.internet.defer.Deferred`.
|
||||
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -155,7 +153,7 @@ item_scraped
|
|||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param response: the response from where the item was scraped
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -175,7 +173,7 @@ item_dropped
|
|||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param response: the response from where the item was dropped
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -203,7 +201,7 @@ item_error
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
|
@ -223,7 +221,7 @@ spider_closed
|
|||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
:param reason: a string which describes the reason why the spider was closed. If
|
||||
it was closed because the spider has completed scraping, the reason
|
||||
|
|
@ -247,7 +245,7 @@ spider_opened
|
|||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
spider_idle
|
||||
~~~~~~~~~~~
|
||||
|
|
@ -268,10 +266,17 @@ spider_idle
|
|||
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
|
||||
prevent the spider from being closed.
|
||||
|
||||
Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
|
||||
exception to provide a custom spider closing reason. An
|
||||
idle handler is the perfect place to put some code that assesses
|
||||
the final spider results and update the final closing reason
|
||||
accordingly (e.g. setting it to 'too_few_results' instead of
|
||||
'finished').
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has gone idle
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
|
||||
**not** guarantee that it can prevent the spider from being closed,
|
||||
|
|
@ -296,7 +301,7 @@ spider_error
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
Request signals
|
||||
---------------
|
||||
|
|
@ -307,16 +312,16 @@ request_scheduled
|
|||
.. signal:: request_scheduled
|
||||
.. function:: request_scheduled(request, spider)
|
||||
|
||||
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
|
||||
Sent when the engine schedules a :class:`~scrapy.Request`, to be
|
||||
downloaded later.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached the scheduler
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
request_dropped
|
||||
~~~~~~~~~~~~~~~
|
||||
|
|
@ -324,16 +329,16 @@ request_dropped
|
|||
.. signal:: request_dropped
|
||||
.. function:: request_dropped(request, spider)
|
||||
|
||||
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
|
||||
Sent when a :class:`~scrapy.Request`, scheduled by the engine to be
|
||||
downloaded later, is rejected by the scheduler.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached the scheduler
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
request_reached_downloader
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -341,15 +346,15 @@ request_reached_downloader
|
|||
.. signal:: request_reached_downloader
|
||||
.. function:: request_reached_downloader(request, spider)
|
||||
|
||||
Sent when a :class:`~scrapy.http.Request` reached downloader.
|
||||
Sent when a :class:`~scrapy.Request` reached downloader.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached downloader
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
request_left_downloader
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -359,16 +364,16 @@ request_left_downloader
|
|||
|
||||
.. versionadded:: 2.0
|
||||
|
||||
Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
|
||||
Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of
|
||||
failure.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached the downloader
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
bytes_received
|
||||
~~~~~~~~~~~~~~
|
||||
|
|
@ -384,22 +389,52 @@ bytes_received
|
|||
a possible scenario for a 25 kb response would be two signals fired
|
||||
with 10 kb of data, and a final one with 5 kb of data.
|
||||
|
||||
Handlers for this signal can stop the download of a response while it
|
||||
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
|
||||
exception. Please refer to the :ref:`topics-stop-response-download` topic
|
||||
for additional information and examples.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param data: the data received by the download handler
|
||||
:type data: :class:`bytes` object
|
||||
|
||||
:param request: the request that generated the download
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider associated with the response
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. note:: Handlers of this signal can stop the download of a response while it
|
||||
headers_received
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.5
|
||||
|
||||
.. signal:: headers_received
|
||||
.. function:: headers_received(headers, body_length, request, spider)
|
||||
|
||||
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||
available for a given request, before downloading any additional content.
|
||||
|
||||
Handlers for this signal can stop the download of a response while it
|
||||
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
|
||||
exception. Please refer to the :ref:`topics-stop-response-download` topic
|
||||
for additional information and examples.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param headers: the headers received by the download handler
|
||||
:type headers: :class:`scrapy.http.headers.Headers` object
|
||||
|
||||
:param body_length: expected size of the response body, in bytes
|
||||
:type body_length: `int`
|
||||
|
||||
:param request: the request that generated the download
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider associated with the response
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
Response signals
|
||||
----------------
|
||||
|
||||
|
|
@ -418,10 +453,10 @@ response_received
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param request: the request that generated the response
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. note:: The ``request`` argument might not contain the original request that
|
||||
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
|
||||
|
|
@ -442,7 +477,7 @@ response_downloaded
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param request: the request that generated the response
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider for which this response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
|
||||
.. method:: process_spider_output(response, result, spider)
|
||||
|
|
@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
it has processed the response.
|
||||
|
||||
:meth:`process_spider_output` must return an iterable of
|
||||
:class:`~scrapy.http.Request` objects and :ref:`item object
|
||||
:class:`~scrapy.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
|
||||
:param response: the response which generated this output from the
|
||||
|
|
@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param result: the result returned by the spider
|
||||
:type result: an iterable of :class:`~scrapy.http.Request` objects and
|
||||
:type result: an iterable of :class:`~scrapy.Request` objects and
|
||||
:ref:`item object <topics-items>`
|
||||
|
||||
:param spider: the spider whose result is being processed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: process_spider_exception(response, exception, spider)
|
||||
|
||||
|
|
@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
method (from a previous spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
|
||||
objects.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
executing any other :meth:`process_spider_exception` in the following
|
||||
|
|
@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type exception: :exc:`Exception` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: process_start_requests(start_requests, spider)
|
||||
|
||||
|
|
@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
items).
|
||||
|
||||
It receives an iterable (in the ``start_requests`` parameter) and must
|
||||
return another iterable of :class:`~scrapy.http.Request` objects.
|
||||
return another iterable of :class:`~scrapy.Request` objects.
|
||||
|
||||
.. note:: When implementing this method in your spider middleware, you
|
||||
should always return an iterable (that follows the input one) and
|
||||
|
|
@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
(like a time limit or item/page count).
|
||||
|
||||
:param start_requests: the start requests
|
||||
:type start_requests: an iterable of :class:`~scrapy.http.Request`
|
||||
:type start_requests: an iterable of :class:`~scrapy.Request`
|
||||
|
||||
:param spider: the spider to whom the start requests belong
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
|
|
@ -251,9 +251,10 @@ this::
|
|||
.. reqmeta:: handle_httpstatus_all
|
||||
|
||||
The ``handle_httpstatus_list`` key of :attr:`Request.meta
|
||||
<scrapy.http.Request.meta>` can also be used to specify which response codes to
|
||||
<scrapy.Request.meta>` can also be used to specify which response codes to
|
||||
allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
|
||||
to ``True`` if you want to allow any response code for a request.
|
||||
to ``True`` if you want to allow any response code for a request, and ``False`` to
|
||||
disable the effects of the ``handle_httpstatus_all`` key.
|
||||
|
||||
Keep in mind, however, that it's usually a bad idea to handle non-200
|
||||
responses, unless you really know what you're doing.
|
||||
|
|
@ -294,7 +295,7 @@ OffsiteMiddleware
|
|||
Filters out Requests for URLs outside the domains covered by the spider.
|
||||
|
||||
This middleware filters out every request whose host names aren't in the
|
||||
spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
|
||||
spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
|
||||
All subdomains of any domain in the list are also allowed.
|
||||
E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
|
||||
but not ``www2.example.com`` nor ``example.com``.
|
||||
|
|
@ -312,10 +313,10 @@ OffsiteMiddleware
|
|||
will be printed (but only for the first request filtered).
|
||||
|
||||
If the spider doesn't define an
|
||||
:attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the
|
||||
:attr:`~scrapy.Spider.allowed_domains` attribute, or the
|
||||
attribute is empty, the offsite middleware will allow all requests.
|
||||
|
||||
If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute
|
||||
If the request has the :attr:`~scrapy.Request.dont_filter` attribute
|
||||
set, the offsite middleware will allow the request even if its domain is not
|
||||
listed in allowed domains.
|
||||
|
||||
|
|
@ -439,4 +440,3 @@ UrlLengthMiddleware
|
|||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this:
|
|||
those requests.
|
||||
|
||||
The first requests to perform are obtained by calling the
|
||||
:meth:`~scrapy.spiders.Spider.start_requests` method which (by default)
|
||||
generates :class:`~scrapy.http.Request` for the URLs specified in the
|
||||
:attr:`~scrapy.spiders.Spider.start_urls` and the
|
||||
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
|
||||
:meth:`~scrapy.Spider.start_requests` method which (by default)
|
||||
generates :class:`~scrapy.Request` for the URLs specified in the
|
||||
:attr:`~scrapy.Spider.start_urls` and the
|
||||
:attr:`~scrapy.Spider.parse` method as callback function for the
|
||||
Requests.
|
||||
|
||||
2. In the callback function, you parse the response (web page) and return
|
||||
:ref:`item objects <topics-items>`,
|
||||
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
|
||||
:class:`~scrapy.Request` objects, or an iterable of these objects.
|
||||
Those Requests will also contain a callback (maybe
|
||||
the same) and will then be downloaded by Scrapy and then their
|
||||
response handled by the specified callback.
|
||||
|
|
@ -42,15 +42,13 @@ Even though this cycle applies (more or less) to any kind of spider, there are
|
|||
different kinds of default spiders bundled into Scrapy for different purposes.
|
||||
We will talk about those types here.
|
||||
|
||||
.. module:: scrapy.spiders
|
||||
:synopsis: Spiders base class, spider manager and spider middleware
|
||||
|
||||
.. _topics-spiders-ref:
|
||||
|
||||
scrapy.Spider
|
||||
=============
|
||||
|
||||
.. class:: Spider()
|
||||
.. class:: scrapy.spiders.Spider
|
||||
.. class:: scrapy.Spider()
|
||||
|
||||
This is the simplest spider, and the one from which every other spider
|
||||
must inherit (including spiders that come bundled with Scrapy, as well as spiders
|
||||
|
|
@ -86,7 +84,7 @@ scrapy.Spider
|
|||
|
||||
A list of URLs where the spider will begin to crawl from, when no
|
||||
particular URLs are specified. So, the first pages downloaded will be those
|
||||
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
|
||||
listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data
|
||||
contained in the start URLs.
|
||||
|
||||
.. attribute:: custom_settings
|
||||
|
|
@ -121,6 +119,11 @@ scrapy.Spider
|
|||
send log messages through it as described on
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. attribute:: state
|
||||
|
||||
A dict you can use to persist some spider state between batches.
|
||||
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
|
||||
|
||||
.. method:: from_crawler(crawler, *args, **kwargs)
|
||||
|
||||
This is the class method used by Scrapy to create your spiders.
|
||||
|
|
@ -179,7 +182,7 @@ scrapy.Spider
|
|||
the same requirements as the :class:`Spider` class.
|
||||
|
||||
This method, as well as any other Request callback, must return an
|
||||
iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
|
||||
iterable of :class:`~scrapy.Request` and/or :ref:`item objects
|
||||
<topics-items>`.
|
||||
|
||||
:param response: the response to parse
|
||||
|
|
@ -234,7 +237,7 @@ Return multiple Requests and items from a single callback::
|
|||
yield scrapy.Request(response.urljoin(href), self.parse)
|
||||
|
||||
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
|
||||
to give data more structure you can use :class:`~scrapy.item.Item` objects::
|
||||
to give data more structure you can use :class:`~scrapy.Item` objects::
|
||||
|
||||
import scrapy
|
||||
from myproject.items import MyItem
|
||||
|
|
@ -294,6 +297,14 @@ The above example can also be written as follows::
|
|||
def start_requests(self):
|
||||
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
|
||||
|
||||
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
||||
specify spider arguments when calling
|
||||
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
|
||||
|
||||
process = CrawlerProcess()
|
||||
process.crawl(MySpider, category="electronics")
|
||||
|
||||
Keep in mind that spider arguments are only strings.
|
||||
The spider will not do any parsing on its own.
|
||||
If you were to set the ``start_urls`` attribute from the command line,
|
||||
|
|
@ -358,14 +369,14 @@ CrawlSpider
|
|||
described below. If multiple rules match the same link, the first one
|
||||
will be used, according to the order they're defined in this attribute.
|
||||
|
||||
This spider also exposes an overrideable method:
|
||||
This spider also exposes an overridable method:
|
||||
|
||||
.. method:: parse_start_url(response, **kwargs)
|
||||
|
||||
This method is called for each response produced for the URLs in
|
||||
the spider's ``start_urls`` attribute. It allows to parse
|
||||
the initial responses and must return either an
|
||||
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
|
||||
:ref:`item object <topics-items>`, a :class:`~scrapy.Request`
|
||||
object, or an iterable containing any of them.
|
||||
|
||||
Crawling rules
|
||||
|
|
@ -375,7 +386,7 @@ Crawling rules
|
|||
|
||||
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
|
||||
defines how links will be extracted from each crawled page. Each produced link will
|
||||
be used to generate a :class:`~scrapy.http.Request` object, which will contain the
|
||||
be used to generate a :class:`~scrapy.Request` object, which will contain the
|
||||
link's text in its ``meta`` dictionary (under the ``link_text`` key).
|
||||
If omitted, a default link extractor created with no arguments will be used,
|
||||
resulting in all links being extracted.
|
||||
|
|
@ -384,9 +395,9 @@ Crawling rules
|
|||
object with that name will be used) to be called for each link extracted with
|
||||
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
|
||||
as its first argument and must return either a single instance or an iterable of
|
||||
:ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
|
||||
:ref:`item objects <topics-items>` and/or :class:`~scrapy.Request` objects
|
||||
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
|
||||
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
|
||||
object will contain the text of the link that produced the :class:`~scrapy.Request`
|
||||
in its ``meta`` dictionary (under the ``link_text`` key)
|
||||
|
||||
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
|
||||
|
|
@ -403,7 +414,7 @@ Crawling rules
|
|||
|
||||
``process_request`` is a callable (or a string, in which case a method from
|
||||
the spider object with that name will be used) which will be called for every
|
||||
:class:`~scrapy.http.Request` extracted by this rule. This callable should
|
||||
:class:`~scrapy.Request` extracted by this rule. This callable should
|
||||
take said request as first argument and the :class:`~scrapy.http.Response`
|
||||
from which the request originated as second argument. It must return a
|
||||
``Request`` object or ``None`` (to filter out the request).
|
||||
|
|
@ -414,10 +425,9 @@ Crawling rules
|
|||
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
|
||||
instance as first parameter.
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
.. versionadded:: 2.0
|
||||
The *errback* parameter.
|
||||
|
|
@ -463,7 +473,7 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
This spider would start crawling example.com's home page, collecting category
|
||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||
each item response, some data will be extracted from the HTML using XPath, and
|
||||
an :class:`~scrapy.item.Item` will be filled with it.
|
||||
an :class:`~scrapy.Item` will be filled with it.
|
||||
|
||||
XMLFeedSpider
|
||||
-------------
|
||||
|
|
@ -486,11 +496,11 @@ XMLFeedSpider
|
|||
|
||||
- ``'iternodes'`` - a fast iterator based on regular expressions
|
||||
|
||||
- ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
|
||||
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
|
||||
Keep in mind this uses DOM parsing and must load all DOM in memory
|
||||
which could be a problem for big feeds
|
||||
|
||||
- ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
|
||||
- ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`.
|
||||
Keep in mind this uses DOM parsing and must load all DOM in memory
|
||||
which could be a problem for big feeds
|
||||
|
||||
|
|
@ -508,7 +518,7 @@ XMLFeedSpider
|
|||
available in that document that will be processed with this spider. The
|
||||
``prefix`` and ``uri`` will be used to automatically register
|
||||
namespaces using the
|
||||
:meth:`~scrapy.selector.Selector.register_namespace` method.
|
||||
:meth:`~scrapy.Selector.register_namespace` method.
|
||||
|
||||
You can then specify nodes with namespaces in the :attr:`itertag`
|
||||
attribute.
|
||||
|
|
@ -521,7 +531,7 @@ XMLFeedSpider
|
|||
itertag = 'n:url'
|
||||
# ...
|
||||
|
||||
Apart from these new attributes, this spider has the following overrideable
|
||||
Apart from these new attributes, this spider has the following overridable
|
||||
methods too:
|
||||
|
||||
.. method:: adapt_response(response)
|
||||
|
|
@ -535,10 +545,10 @@ XMLFeedSpider
|
|||
|
||||
This method is called for the nodes matching the provided tag name
|
||||
(``itertag``). Receives the response and an
|
||||
:class:`~scrapy.selector.Selector` for each node. Overriding this
|
||||
:class:`~scrapy.Selector` for each node. Overriding this
|
||||
method is mandatory. Otherwise, you spider won't work. This method
|
||||
must return an :ref:`item object <topics-items>`, a
|
||||
:class:`~scrapy.http.Request` object, or an iterable containing any of
|
||||
:class:`~scrapy.Request` object, or an iterable containing any of
|
||||
them.
|
||||
|
||||
.. method:: process_results(response, results)
|
||||
|
|
@ -549,10 +559,9 @@ XMLFeedSpider
|
|||
item IDs. It receives a list of results and the response which originated
|
||||
those results. It must return a list of results (items or requests).
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
|
||||
XMLFeedSpider example
|
||||
|
|
@ -581,7 +590,7 @@ These spiders are pretty easy to use, let's have a look at one example::
|
|||
|
||||
Basically what we did up there was to create a spider that downloads a feed from
|
||||
the given ``start_urls``, and then iterates through each of its ``item`` tags,
|
||||
prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
|
||||
prints them out, and stores some random data in an :class:`~scrapy.Item`.
|
||||
|
||||
CSVFeedSpider
|
||||
-------------
|
||||
|
|
|
|||
|
|
@ -110,11 +110,10 @@ using the telnet console::
|
|||
Execution engine status
|
||||
|
||||
time()-engine.start_time : 8.62972998619
|
||||
engine.has_capacity() : False
|
||||
len(engine.downloader.active) : 16
|
||||
engine.scraper.is_idle() : False
|
||||
engine.spider.name : followall
|
||||
engine.spider_is_idle(engine.spider) : False
|
||||
engine.spider_is_idle() : False
|
||||
engine.slot.closing : False
|
||||
len(engine.slot.inprogress) : 16
|
||||
len(engine.slot.scheduler.dqs or []) : 0
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
|
|||
large changes.
|
||||
* *B* is the release number. This will include many changes including features
|
||||
and things that possibly break backward compatibility, although we strive to
|
||||
keep theses cases at a minimum.
|
||||
keep these cases at a minimum.
|
||||
* *C* is the bugfix release number.
|
||||
|
||||
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
A spider that generate light requests to meassure QPS troughput
|
||||
A spider that generate light requests to meassure QPS throughput
|
||||
|
||||
usage:
|
||||
|
||||
|
|
|
|||
7
pylintrc
7
pylintrc
|
|
@ -6,6 +6,7 @@ jobs=1 # >1 hides results
|
|||
disable=abstract-method,
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
arguments-renamed,
|
||||
attribute-defined-outside-init,
|
||||
bad-classmethod-argument,
|
||||
bad-continuation,
|
||||
|
|
@ -21,9 +22,12 @@ disable=abstract-method,
|
|||
cell-var-from-loop,
|
||||
comparison-with-callable,
|
||||
consider-iterating-dictionary,
|
||||
consider-using-dict-items,
|
||||
consider-using-from-import,
|
||||
consider-using-in,
|
||||
consider-using-set-comprehension,
|
||||
consider-using-sys-exit,
|
||||
consider-using-with,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
deprecated-method,
|
||||
|
|
@ -101,11 +105,14 @@ disable=abstract-method,
|
|||
unnecessary-lambda,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unspecified-encoding,
|
||||
unsubscriptable-object,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-private-member,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
use-implicit-booleaness-not-comparison,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
|
|
|
|||
22
pytest.ini
22
pytest.ini
|
|
@ -18,26 +18,6 @@ addopts =
|
|||
--ignore=docs/topics/stats.rst
|
||||
--ignore=docs/topics/telnetconsole.rst
|
||||
--ignore=docs/utils
|
||||
twisted = 1
|
||||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
flake8-max-line-length = 119
|
||||
flake8-ignore =
|
||||
W503
|
||||
|
||||
# Exclude files that are meant to provide top-level imports
|
||||
# E402: Module level import not at top of file
|
||||
# F401: Module imported but unused
|
||||
scrapy/__init__.py E402
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/linkextractors/__init__.py E402 F401
|
||||
scrapy/selector/__init__.py F401
|
||||
scrapy/spiders/__init__.py E402 F401
|
||||
|
||||
# Issues pending a review:
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/markup.py F403
|
||||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.4.1
|
||||
2.6.1
|
||||
|
|
|
|||
|
|
@ -22,14 +22,14 @@ __all__ = [
|
|||
|
||||
|
||||
# Scrapy and Twisted versions
|
||||
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
|
||||
__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
|
||||
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
|
||||
twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
||||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 6):
|
||||
print("Scrapy %s requires Python 3.6+" % __version__)
|
||||
if sys.version_info < (3, 7):
|
||||
print(f"Scrapy {__version__} requires Python 3.7+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import argparse
|
||||
import cProfile
|
||||
import inspect
|
||||
import pkg_resources
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
|
|
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
|
||||
parser.description = cmd.long_desc()
|
||||
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler='resolve',
|
||||
description=cmd.long_desc())
|
||||
settings.setdict(cmd.default_settings, priority='command')
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
cmd.crawler_process = CrawlerProcess(settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from optparse import OptionGroup
|
||||
import argparse
|
||||
from typing import Any, Dict
|
||||
|
||||
from twisted.python import failure
|
||||
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
|
|
@ -15,7 +17,7 @@ class ScrapyCommand:
|
|||
crawler_process = None
|
||||
|
||||
# default settings to be used for this command instead of global defaults
|
||||
default_settings = {}
|
||||
default_settings: Dict[str, Any] = {}
|
||||
|
||||
exitcode = 0
|
||||
|
||||
|
|
@ -41,14 +43,14 @@ class ScrapyCommand:
|
|||
|
||||
def long_desc(self):
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since no post-formatting will
|
||||
"help" command. It can contain newlines since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
|
@ -57,22 +59,20 @@ class ScrapyCommand:
|
|||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
group = parser.add_argument_group(title='Global Options')
|
||||
group.add_argument("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_argument("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_argument("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_argument("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
try:
|
||||
|
|
@ -112,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_argument("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_argument("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -135,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
|
||||
class ScrapyHelpFormatter(argparse.HelpFormatter):
|
||||
"""
|
||||
Help Formatter for scrapy command line help messages.
|
||||
"""
|
||||
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
|
||||
super().__init__(prog, indent_increment=indent_increment,
|
||||
max_help_position=max_help_position, width=width)
|
||||
|
||||
def _join_parts(self, part_strings):
|
||||
parts = self.format_part_strings(part_strings)
|
||||
return super()._join_parts(parts)
|
||||
|
||||
def format_part_strings(self, part_strings):
|
||||
"""
|
||||
Underline and title case command line help message headers.
|
||||
"""
|
||||
if part_strings and part_strings[0].startswith("usage: "):
|
||||
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
|
||||
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
|
||||
for index in headings[::-1]:
|
||||
char = '-' if "Global Options" in part_strings[index] else '='
|
||||
part_strings[index] = part_strings[index][:-2].title()
|
||||
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
|
||||
part_strings.insert(index + 1, underline)
|
||||
return part_strings
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
|
|||
|
||||
def start_requests(self):
|
||||
qargs = {'total': self.total, 'show': self.show}
|
||||
url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
|
||||
url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
|
||||
return [scrapy.Request(url, dont_filter=True)]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
|
|||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
elif len(args) > 1:
|
||||
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
|
||||
raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
|
||||
spname = args[0]
|
||||
|
||||
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
parser.add_argument("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import string
|
|||
|
||||
from importlib import import_module
|
||||
from os.path import join, dirname, abspath, exists, splitext
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
|
|||
return module_name
|
||||
|
||||
|
||||
def extract_domain(url):
|
||||
"""Extract domain name from URL string"""
|
||||
o = urlparse(url)
|
||||
if o.scheme == '' and o.netloc == '':
|
||||
o = urlparse("//" + url.lstrip("/"))
|
||||
return o.netloc
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
@ -35,16 +44,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_argument("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_argument("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
@ -59,7 +68,8 @@ class Command(ScrapyCommand):
|
|||
if len(args) != 2:
|
||||
raise UsageError()
|
||||
|
||||
name, domain = args[0:2]
|
||||
name, url = args[0:2]
|
||||
domain = extract_domain(url)
|
||||
module = sanitize_module_name(name)
|
||||
|
||||
if self.settings.get('BOT_NAME') == module:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
from w3lib.url import is_url
|
||||
|
|
@ -10,6 +11,7 @@ from scrapy.utils import display
|
|||
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):
|
|||
requires_project = True
|
||||
|
||||
spider = None
|
||||
items = {}
|
||||
requests = {}
|
||||
items: Dict[int, list] = {}
|
||||
requests: Dict[int, list] = {}
|
||||
|
||||
first_response = None
|
||||
|
||||
|
|
@ -30,28 +32,28 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
parser.add_argument("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_argument("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_argument("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_argument("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_argument("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
@ -144,7 +146,8 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def _start_requests(spider):
|
||||
yield self.prepare_request(spider, Request(url), opts)
|
||||
self.spidercls.start_requests = _start_requests
|
||||
if self.spidercls:
|
||||
self.spidercls.start_requests = _start_requests
|
||||
|
||||
def start_parsing(self, url, opts):
|
||||
self.crawler_process.crawl(self.spidercls, **opts.spargs)
|
||||
|
|
|
|||
|
|
@ -18,16 +18,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
parser.add_argument("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_argument("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_argument("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _start_crawler_thread(self):
|
||||
t = Thread(target=self.crawler_process.start,
|
||||
kwargs={'stop_after_crawl': False})
|
||||
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import re
|
||||
import os
|
||||
import string
|
||||
from importlib import import_module
|
||||
from importlib.util import find_spec
|
||||
from os.path import join, exists, abspath
|
||||
from shutil import ignore_patterns, move, copy2, copystat
|
||||
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
|
||||
|
|
@ -42,11 +42,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _is_valid_name(self, project_name):
|
||||
def _module_exists(module_name):
|
||||
try:
|
||||
import_module(module_name)
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
spec = find_spec(module_name)
|
||||
return spec is not None and spec.loader is not None
|
||||
|
||||
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
|
||||
print('Error: Project names must begin with a letter and contain'
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ class Command(fetch.Command):
|
|||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
parser.add_argument('--headers', help=argparse.SUPPRESS)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,77 @@
|
|||
import sys
|
||||
import re
|
||||
import sys
|
||||
from functools import wraps
|
||||
from inspect import getmembers
|
||||
from typing import Dict
|
||||
from unittest import TestCase
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
from scrapy.utils.python import get_spec
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
class Contract:
|
||||
""" Abstract class for contracts """
|
||||
request_cls = None
|
||||
|
||||
def __init__(self, method, *args):
|
||||
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
|
||||
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
|
||||
self.args = args
|
||||
|
||||
def add_pre_hook(self, request, results):
|
||||
if hasattr(self, 'pre_process'):
|
||||
cb = request.callback
|
||||
|
||||
@wraps(cb)
|
||||
def wrapper(response, **cb_kwargs):
|
||||
try:
|
||||
results.startTest(self.testcase_pre)
|
||||
self.pre_process(response)
|
||||
results.stopTest(self.testcase_pre)
|
||||
except AssertionError:
|
||||
results.addFailure(self.testcase_pre, sys.exc_info())
|
||||
except Exception:
|
||||
results.addError(self.testcase_pre, sys.exc_info())
|
||||
else:
|
||||
results.addSuccess(self.testcase_pre)
|
||||
finally:
|
||||
return list(iterate_spider_output(cb(response, **cb_kwargs)))
|
||||
|
||||
request.callback = wrapper
|
||||
|
||||
return request
|
||||
|
||||
def add_post_hook(self, request, results):
|
||||
if hasattr(self, 'post_process'):
|
||||
cb = request.callback
|
||||
|
||||
@wraps(cb)
|
||||
def wrapper(response, **cb_kwargs):
|
||||
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
|
||||
try:
|
||||
results.startTest(self.testcase_post)
|
||||
self.post_process(output)
|
||||
results.stopTest(self.testcase_post)
|
||||
except AssertionError:
|
||||
results.addFailure(self.testcase_post, sys.exc_info())
|
||||
except Exception:
|
||||
results.addError(self.testcase_post, sys.exc_info())
|
||||
else:
|
||||
results.addSuccess(self.testcase_post)
|
||||
finally:
|
||||
return output
|
||||
|
||||
request.callback = wrapper
|
||||
|
||||
return request
|
||||
|
||||
def adjust_request_args(self, args):
|
||||
return args
|
||||
|
||||
|
||||
class ContractsManager:
|
||||
contracts = {}
|
||||
contracts: Dict[str, Contract] = {}
|
||||
|
||||
def __init__(self, contracts):
|
||||
for contract in contracts:
|
||||
|
|
@ -107,66 +168,6 @@ class ContractsManager:
|
|||
request.errback = eb_wrapper
|
||||
|
||||
|
||||
class Contract:
|
||||
""" Abstract class for contracts """
|
||||
request_cls = None
|
||||
|
||||
def __init__(self, method, *args):
|
||||
self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
|
||||
self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
|
||||
self.args = args
|
||||
|
||||
def add_pre_hook(self, request, results):
|
||||
if hasattr(self, 'pre_process'):
|
||||
cb = request.callback
|
||||
|
||||
@wraps(cb)
|
||||
def wrapper(response, **cb_kwargs):
|
||||
try:
|
||||
results.startTest(self.testcase_pre)
|
||||
self.pre_process(response)
|
||||
results.stopTest(self.testcase_pre)
|
||||
except AssertionError:
|
||||
results.addFailure(self.testcase_pre, sys.exc_info())
|
||||
except Exception:
|
||||
results.addError(self.testcase_pre, sys.exc_info())
|
||||
else:
|
||||
results.addSuccess(self.testcase_pre)
|
||||
finally:
|
||||
return list(iterate_spider_output(cb(response, **cb_kwargs)))
|
||||
|
||||
request.callback = wrapper
|
||||
|
||||
return request
|
||||
|
||||
def add_post_hook(self, request, results):
|
||||
if hasattr(self, 'post_process'):
|
||||
cb = request.callback
|
||||
|
||||
@wraps(cb)
|
||||
def wrapper(response, **cb_kwargs):
|
||||
output = list(iterate_spider_output(cb(response, **cb_kwargs)))
|
||||
try:
|
||||
results.startTest(self.testcase_post)
|
||||
self.post_process(output)
|
||||
results.stopTest(self.testcase_post)
|
||||
except AssertionError:
|
||||
results.addFailure(self.testcase_post, sys.exc_info())
|
||||
except Exception:
|
||||
results.addError(self.testcase_post, sys.exc_info())
|
||||
else:
|
||||
results.addSuccess(self.testcase_post)
|
||||
finally:
|
||||
return output
|
||||
|
||||
request.callback = wrapper
|
||||
|
||||
return request
|
||||
|
||||
def adjust_request_args(self, args):
|
||||
return args
|
||||
|
||||
|
||||
def _create_testcase(method, desc):
|
||||
spider = method.__self__.name
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
import warnings
|
||||
|
||||
from OpenSSL import SSL
|
||||
from twisted.internet._sslverify import _setAcceptableProtocols
|
||||
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
from zope.interface.declarations import implementer
|
||||
from zope.interface.verify import verifyObject
|
||||
|
||||
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
|
||||
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
|
|
@ -81,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
The default OpenSSL method is ``TLS_METHOD`` (also called
|
||||
``SSLv23_METHOD``) which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
# trustRoot set to platformTrust() will use the platform's root CAs.
|
||||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
|
|
@ -92,3 +97,51 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={'method': self._ssl_method},
|
||||
)
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class AcceptableProtocolsContextFactory:
|
||||
"""Context factory to used to override the acceptable protocols
|
||||
to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
|
||||
negotiation.
|
||||
"""
|
||||
|
||||
def __init__(self, context_factory, acceptable_protocols):
|
||||
verifyObject(IPolicyForHTTPS, context_factory)
|
||||
self._wrapped_context_factory = context_factory
|
||||
self._acceptable_protocols = acceptable_protocols
|
||||
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
|
||||
_setAcceptableProtocols(options._ctx, self._acceptable_protocols)
|
||||
return options
|
||||
|
||||
|
||||
def load_context_factory_from_settings(settings, crawler):
|
||||
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
|
||||
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
|
||||
# try method-aware context factory
|
||||
try:
|
||||
context_factory = create_instance(
|
||||
objcls=context_factory_cls,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
method=ssl_method,
|
||||
)
|
||||
except TypeError:
|
||||
# use context factory defaults
|
||||
context_factory = create_instance(
|
||||
objcls=context_factory_cls,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
)
|
||||
msg = (
|
||||
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
|
||||
"a `method` argument (type OpenSSL.SSL method, e.g. "
|
||||
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
|
||||
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
|
||||
"context factory class to handle them or ignore them."
|
||||
)
|
||||
warnings.warn(msg)
|
||||
|
||||
return context_factory
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import warnings
|
|||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from time import time
|
||||
from urllib.parse import urldefrag
|
||||
from urllib.parse import urldefrag, urlunparse
|
||||
|
||||
from twisted.internet import defer, protocol, ssl
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
|
|
@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
|
|||
from zope.interface import implementer
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.core.downloader.tls import openssl_methods
|
||||
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
|
||||
|
|
@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
|
|||
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
|
||||
self._pool._factory.noisy = False
|
||||
|
||||
self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
|
||||
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
|
||||
# try method-aware context factory
|
||||
try:
|
||||
self._contextFactory = create_instance(
|
||||
objcls=self._contextFactoryClass,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
method=self._sslMethod,
|
||||
)
|
||||
except TypeError:
|
||||
# use context factory defaults
|
||||
self._contextFactory = create_instance(
|
||||
objcls=self._contextFactoryClass,
|
||||
settings=settings,
|
||||
crawler=crawler,
|
||||
)
|
||||
msg = f"""
|
||||
'{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \
|
||||
argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \
|
||||
`tls_verbose_logging` argument and/or `tls_ciphers` argument.\
|
||||
Please upgrade your context factory class to handle them or ignore them."""
|
||||
warnings.warn(msg)
|
||||
self._contextFactory = load_context_factory_from_settings(settings, crawler)
|
||||
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
|
||||
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
|
||||
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
|
||||
|
|
@ -121,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
with this endpoint comes from the pool and a CONNECT has already been issued
|
||||
for it.
|
||||
"""
|
||||
|
||||
_responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
|
||||
_truncatedLength = 1000
|
||||
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
|
||||
_responseMatcher = re.compile(_responseAnswer.encode())
|
||||
|
||||
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
|
||||
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
|
||||
|
|
@ -167,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
extra = {'status': int(respm.group('status')),
|
||||
'reason': respm.group('reason').strip()}
|
||||
else:
|
||||
extra = rcvd_bytes[:32]
|
||||
extra = rcvd_bytes[:self._truncatedLength]
|
||||
self._tunnelReadyDeferred.errback(
|
||||
TunnelError('Could not open CONNECT tunnel with proxy '
|
||||
f'{self._host}:{self._port} [{extra!r}]')
|
||||
|
|
@ -235,7 +213,7 @@ class TunnelingAgent(Agent):
|
|||
# proxy host and port are required for HTTP pool `key`
|
||||
# otherwise, same remote host connection request could reuse
|
||||
# a cached tunneled connection to a different proxy
|
||||
key = key + self._proxyConf
|
||||
key += self._proxyConf
|
||||
return super()._requestWithEndpoint(
|
||||
key=key,
|
||||
endpoint=endpoint,
|
||||
|
|
@ -298,16 +276,19 @@ class ScrapyAgent:
|
|||
bindaddress = request.meta.get('bindaddress') or self._bindAddress
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
|
||||
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
|
||||
scheme = _parse(request.url)[0]
|
||||
proxyHost = to_unicode(proxyHost)
|
||||
omitConnectTunnel = b'noconnect' in proxyParams
|
||||
if omitConnectTunnel:
|
||||
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Crawlera, it doesn't require this mode anymore, "
|
||||
"so you should update scrapy-crawlera to 1.3.0+ "
|
||||
"and remove '?noconnect' from the Crawlera URL.",
|
||||
ScrapyDeprecationWarning)
|
||||
warnings.warn(
|
||||
"Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Zyte Smart Proxy Manager, it doesn't require "
|
||||
"this mode anymore, so you should update scrapy-crawlera "
|
||||
"to scrapy-zyte-smartproxy and remove '?noconnect' "
|
||||
"from the Zyte Smart Proxy Manager URL.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
if scheme == b'https' and not omitConnectTunnel:
|
||||
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
|
||||
proxyConf = (proxyHost, proxyPort, proxyAuth)
|
||||
|
|
@ -320,9 +301,13 @@ class ScrapyAgent:
|
|||
pool=self._pool,
|
||||
)
|
||||
else:
|
||||
proxyScheme = proxyScheme or b'http'
|
||||
proxyHost = to_bytes(proxyHost, encoding='ascii')
|
||||
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
|
||||
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
|
||||
return self._ProxyAgent(
|
||||
reactor=reactor,
|
||||
proxyURI=to_bytes(proxy, encoding='ascii'),
|
||||
proxyURI=to_bytes(proxyURI, encoding='ascii'),
|
||||
connectTimeout=timeout,
|
||||
bindAddress=bindaddress,
|
||||
pool=self._pool,
|
||||
|
|
@ -378,7 +363,38 @@ class ScrapyAgent:
|
|||
request.meta['download_latency'] = time() - start_time
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _headers_from_twisted_response(response):
|
||||
headers = Headers()
|
||||
if response.length != UNKNOWN_LENGTH:
|
||||
headers[b'Content-Length'] = str(response.length).encode()
|
||||
headers.update(response.headers.getAllRawHeaders())
|
||||
return headers
|
||||
|
||||
def _cb_bodyready(self, txresponse, request):
|
||||
headers_received_result = self._crawler.signals.send_catch_log(
|
||||
signal=signals.headers_received,
|
||||
headers=self._headers_from_twisted_response(txresponse),
|
||||
body_length=txresponse.length,
|
||||
request=request,
|
||||
spider=self._crawler.spider,
|
||||
)
|
||||
for handler, result in headers_received_result:
|
||||
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
|
||||
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
|
||||
{"request": request, "handler": handler.__qualname__})
|
||||
txresponse._transport.stopProducing()
|
||||
with suppress(AttributeError):
|
||||
txresponse._transport._producer.loseConnection()
|
||||
return {
|
||||
"txresponse": txresponse,
|
||||
"body": b"",
|
||||
"flags": ["download_stopped"],
|
||||
"certificate": None,
|
||||
"ip_address": None,
|
||||
"failure": result if result.value.fail else None,
|
||||
}
|
||||
|
||||
# deliverBody hangs for responses without body
|
||||
if txresponse.length == 0:
|
||||
return {
|
||||
|
|
@ -432,8 +448,13 @@ class ScrapyAgent:
|
|||
return d
|
||||
|
||||
def _cb_bodydone(self, result, request, url):
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
headers = self._headers_from_twisted_response(result["txresponse"])
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
try:
|
||||
version = result["txresponse"].version
|
||||
protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
|
||||
except (AttributeError, TypeError, IndexError):
|
||||
protocol = None
|
||||
response = respcls(
|
||||
url=url,
|
||||
status=int(result["txresponse"].code),
|
||||
|
|
@ -442,6 +463,7 @@ class ScrapyAgent:
|
|||
flags=result["flags"],
|
||||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
protocol=protocol,
|
||||
)
|
||||
if result.get("failure"):
|
||||
result["failure"].value.response = response
|
||||
|
|
@ -520,6 +542,7 @@ class _ResponseReader(protocol.Protocol):
|
|||
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
|
||||
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
|
||||
{"request": self._request, "handler": handler.__qualname__})
|
||||
self.transport.stopProducing()
|
||||
self.transport._producer.loseConnection()
|
||||
failure = result if result.value.fail else None
|
||||
self._finish_response(flags=["download_stopped"], failure=failure)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,129 @@
|
|||
import warnings
|
||||
from time import time
|
||||
from typing import Optional, Type, TypeVar
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
from twisted.internet.base import DelayedCall
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.web.client import URI
|
||||
|
||||
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
||||
H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
|
||||
|
||||
|
||||
class H2DownloadHandler:
|
||||
def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
|
||||
self._crawler = crawler
|
||||
|
||||
from twisted.internet import reactor
|
||||
self._pool = H2ConnectionPool(reactor, settings)
|
||||
self._context_factory = load_context_factory_from_settings(settings, crawler)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
|
||||
return cls(crawler.settings, crawler)
|
||||
|
||||
def download_request(self, request: Request, spider: Spider) -> Deferred:
|
||||
agent = ScrapyH2Agent(
|
||||
context_factory=self._context_factory,
|
||||
pool=self._pool,
|
||||
crawler=self._crawler,
|
||||
)
|
||||
return agent.download_request(request, spider)
|
||||
|
||||
def close(self) -> None:
|
||||
self._pool.close_connections()
|
||||
|
||||
|
||||
class ScrapyH2Agent:
|
||||
_Agent = H2Agent
|
||||
_ProxyAgent = ScrapyProxyH2Agent
|
||||
|
||||
def __init__(
|
||||
self, context_factory,
|
||||
pool: H2ConnectionPool,
|
||||
connect_timeout: int = 10,
|
||||
bind_address: Optional[bytes] = None,
|
||||
crawler: Optional[Crawler] = None,
|
||||
) -> None:
|
||||
self._context_factory = context_factory
|
||||
self._connect_timeout = connect_timeout
|
||||
self._bind_address = bind_address
|
||||
self._pool = pool
|
||||
self._crawler = crawler
|
||||
|
||||
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
|
||||
from twisted.internet import reactor
|
||||
bind_address = request.meta.get('bindaddress') or self._bind_address
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
|
||||
scheme = _parse(request.url)[0]
|
||||
proxy_host = proxy_host.decode()
|
||||
omit_connect_tunnel = b'noconnect' in proxy_params
|
||||
if omit_connect_tunnel:
|
||||
warnings.warn(
|
||||
"Using HTTPS proxies in the noconnect mode is not "
|
||||
"supported by the downloader handler. If you use Zyte "
|
||||
"Smart Proxy Manager, it doesn't require this mode "
|
||||
"anymore, so you should update scrapy-crawlera to "
|
||||
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
|
||||
"Zyte Smart Proxy Manager URL."
|
||||
)
|
||||
|
||||
if scheme == b'https' and not omit_connect_tunnel:
|
||||
# ToDo
|
||||
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
|
||||
return self._ProxyAgent(
|
||||
reactor=reactor,
|
||||
context_factory=self._context_factory,
|
||||
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
|
||||
connect_timeout=timeout,
|
||||
bind_address=bind_address,
|
||||
pool=self._pool,
|
||||
)
|
||||
|
||||
return self._Agent(
|
||||
reactor=reactor,
|
||||
context_factory=self._context_factory,
|
||||
connect_timeout=timeout,
|
||||
bind_address=bind_address,
|
||||
pool=self._pool,
|
||||
)
|
||||
|
||||
def download_request(self, request: Request, spider: Spider) -> Deferred:
|
||||
from twisted.internet import reactor
|
||||
timeout = request.meta.get('download_timeout') or self._connect_timeout
|
||||
agent = self._get_agent(request, timeout)
|
||||
|
||||
start_time = time()
|
||||
d = agent.request(request, spider)
|
||||
d.addCallback(self._cb_latency, request, start_time)
|
||||
|
||||
timeout_cl = reactor.callLater(timeout, d.cancel)
|
||||
d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
|
||||
return d
|
||||
|
||||
@staticmethod
|
||||
def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
|
||||
request.meta['download_latency'] = time() - start_time
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
|
||||
if timeout_cl.active():
|
||||
timeout_cl.cancel()
|
||||
return response
|
||||
|
||||
url = urldefrag(request.url)[0]
|
||||
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
|
||||
|
|
@ -1,5 +1,3 @@
|
|||
from urllib.parse import unquote
|
||||
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
|
|
@ -12,6 +10,7 @@ class S3DownloadHandler:
|
|||
def __init__(self, settings, *,
|
||||
crawler=None,
|
||||
aws_access_key_id=None, aws_secret_access_key=None,
|
||||
aws_session_token=None,
|
||||
httpdownloadhandler=HTTPDownloadHandler, **kw):
|
||||
if not is_botocore_available():
|
||||
raise NotConfigured('missing botocore library')
|
||||
|
|
@ -20,6 +19,8 @@ class S3DownloadHandler:
|
|||
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
|
||||
if not aws_secret_access_key:
|
||||
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
|
||||
if not aws_session_token:
|
||||
aws_session_token = settings['AWS_SESSION_TOKEN']
|
||||
|
||||
# If no credentials could be found anywhere,
|
||||
# consider this an anonymous connection request by default;
|
||||
|
|
@ -38,7 +39,7 @@ class S3DownloadHandler:
|
|||
if not self.anon:
|
||||
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
|
||||
self._signer = SignerCls(botocore.credentials.Credentials(
|
||||
aws_access_key_id, aws_secret_access_key))
|
||||
aws_access_key_id, aws_secret_access_key, aws_session_token))
|
||||
|
||||
_http_handler = create_instance(
|
||||
objcls=httpdownloadhandler,
|
||||
|
|
@ -59,7 +60,7 @@ class S3DownloadHandler:
|
|||
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
|
||||
if self.anon:
|
||||
request = request.replace(url=url)
|
||||
elif self._signer is not None:
|
||||
else:
|
||||
import botocore.awsrequest
|
||||
awsrequest = botocore.awsrequest.AWSRequest(
|
||||
method=request.method,
|
||||
|
|
@ -69,14 +70,4 @@ class S3DownloadHandler:
|
|||
self._signer.add_auth(awsrequest)
|
||||
request = request.replace(
|
||||
url=url, headers=awsrequest.headers.items())
|
||||
else:
|
||||
signed_headers = self.conn.make_request(
|
||||
method=request.method,
|
||||
bucket=bucket,
|
||||
key=unquote(p.path),
|
||||
query_args=unquote(p.query),
|
||||
headers=request.headers,
|
||||
data=request.body,
|
||||
)
|
||||
request = request.replace(url=url, headers=signed_headers)
|
||||
return self._download_http(request, spider)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
from twisted.internet import defer
|
||||
from typing import Callable, Union, cast
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
|
|
@ -29,15 +33,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
if hasattr(mw, 'process_exception'):
|
||||
self.methods['process_exception'].appendleft(mw.process_exception)
|
||||
|
||||
def download(self, download_func, request, spider):
|
||||
def download(self, download_func: Callable, request: Request, spider: Spider):
|
||||
@defer.inlineCallbacks
|
||||
def process_request(request):
|
||||
def process_request(request: Request):
|
||||
for method in self.methods['process_request']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
f"Middleware {method.__self__.__class__.__name__}"
|
||||
".process_request must return None, Response or "
|
||||
f"Middleware {method.__qualname__} must return None, Response or "
|
||||
f"Request, got {response.__class__.__name__}"
|
||||
)
|
||||
if response:
|
||||
|
|
@ -45,18 +49,18 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
return (yield download_func(request=request, spider=spider))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_response(response):
|
||||
def process_response(response: Union[Response, Request]):
|
||||
if response is None:
|
||||
raise TypeError("Received None in process_response")
|
||||
elif isinstance(response, Request):
|
||||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
f"Middleware {method.__self__.__class__.__name__}"
|
||||
".process_response must return Response or Request, "
|
||||
f"Middleware {method.__qualname__} must return Response or Request, "
|
||||
f"got {type(response)}"
|
||||
)
|
||||
if isinstance(response, Request):
|
||||
|
|
@ -64,14 +68,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
return response
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_exception(failure):
|
||||
def process_exception(failure: Failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
f"Middleware {method.__self__.__class__.__name__}"
|
||||
".process_exception must return None, Response or "
|
||||
f"Middleware {method.__qualname__} must return None, Response or "
|
||||
f"Request, got {type(response)}"
|
||||
)
|
||||
if response:
|
||||
|
|
|
|||
|
|
@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
verifyHostname(connection, self._hostnameASCII)
|
||||
except (CertificateError, VerificationError) as e:
|
||||
logger.warning(
|
||||
'Remote certificate is not valid for hostname "{}"; {}'.format(
|
||||
self._hostnameASCII, e))
|
||||
'Remote certificate is not valid for hostname "%s"; %s',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
'Ignoring error while verifying certificate '
|
||||
'from host "{}" (exception: {})'.format(
|
||||
self._hostnameASCII, repr(e)))
|
||||
'from host "%s" (exception: %r)',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
|
||||
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
import re
|
||||
from time import time
|
||||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
|
|
@ -32,6 +33,8 @@ def _parse(url):
|
|||
and is ascii-only.
|
||||
"""
|
||||
url = url.strip()
|
||||
if not re.match(r'^\w+://', url):
|
||||
url = '//' + url
|
||||
parsed = urlparse(url)
|
||||
return _parsed_url_args(parsed)
|
||||
|
||||
|
|
@ -110,7 +113,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
|
|
@ -167,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
from twisted.internet import reactor
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
|
|
|||
|
|
@ -1,51 +1,66 @@
|
|||
"""
|
||||
This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
|
||||
This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
|
||||
|
||||
For more information see docs/topics/architecture.rst
|
||||
|
||||
"""
|
||||
import logging
|
||||
import warnings
|
||||
from time import time
|
||||
from typing import Callable, Iterable, Iterator, Optional, Set, Union
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
|
||||
from twisted.internet.task import LoopingCall
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.core.scraper import Scraper
|
||||
from scrapy.exceptions import DontCloseSpider
|
||||
from scrapy.exceptions import (
|
||||
CloseSpider,
|
||||
DontCloseSpider,
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.reactor import CallLaterOnce
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.reactor import CallLaterOnce
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Slot:
|
||||
|
||||
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
|
||||
self.closing = False
|
||||
self.inprogress = set() # requests in progress
|
||||
self.start_requests = iter(start_requests)
|
||||
def __init__(
|
||||
self,
|
||||
start_requests: Iterable,
|
||||
close_if_idle: bool,
|
||||
nextcall: CallLaterOnce,
|
||||
scheduler,
|
||||
) -> None:
|
||||
self.closing: Optional[Deferred] = None
|
||||
self.inprogress: Set[Request] = set()
|
||||
self.start_requests: Optional[Iterator] = iter(start_requests)
|
||||
self.close_if_idle = close_if_idle
|
||||
self.nextcall = nextcall
|
||||
self.scheduler = scheduler
|
||||
self.heartbeat = task.LoopingCall(nextcall.schedule)
|
||||
self.heartbeat = LoopingCall(nextcall.schedule)
|
||||
|
||||
def add_request(self, request):
|
||||
def add_request(self, request: Request) -> None:
|
||||
self.inprogress.add(request)
|
||||
|
||||
def remove_request(self, request):
|
||||
def remove_request(self, request: Request) -> None:
|
||||
self.inprogress.remove(request)
|
||||
self._maybe_fire_closing()
|
||||
|
||||
def close(self):
|
||||
self.closing = defer.Deferred()
|
||||
def close(self) -> Deferred:
|
||||
self.closing = Deferred()
|
||||
self._maybe_fire_closing()
|
||||
return self.closing
|
||||
|
||||
def _maybe_fire_closing(self):
|
||||
if self.closing and not self.inprogress:
|
||||
def _maybe_fire_closing(self) -> None:
|
||||
if self.closing is not None and not self.inprogress:
|
||||
if self.nextcall:
|
||||
self.nextcall.cancel()
|
||||
if self.heartbeat.running:
|
||||
|
|
@ -54,210 +69,236 @@ class Slot:
|
|||
|
||||
|
||||
class ExecutionEngine:
|
||||
|
||||
def __init__(self, crawler, spider_closed_callback):
|
||||
def __init__(self, crawler, spider_closed_callback: Callable) -> None:
|
||||
self.crawler = crawler
|
||||
self.settings = crawler.settings
|
||||
self.signals = crawler.signals
|
||||
self.logformatter = crawler.logformatter
|
||||
self.slot = None
|
||||
self.spider = None
|
||||
self.slot: Optional[Slot] = None
|
||||
self.spider: Optional[Spider] = None
|
||||
self.running = False
|
||||
self.paused = False
|
||||
self.scheduler_cls = load_object(self.settings['SCHEDULER'])
|
||||
self.scheduler_cls = self._get_scheduler_class(crawler.settings)
|
||||
downloader_cls = load_object(self.settings['DOWNLOADER'])
|
||||
self.downloader = downloader_cls(crawler)
|
||||
self.scraper = Scraper(crawler)
|
||||
self._spider_closed_callback = spider_closed_callback
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
"""Start the execution engine"""
|
||||
def _get_scheduler_class(self, settings: BaseSettings) -> type:
|
||||
from scrapy.core.scheduler import BaseScheduler
|
||||
scheduler_cls = load_object(settings["SCHEDULER"])
|
||||
if not issubclass(scheduler_cls, BaseScheduler):
|
||||
raise TypeError(
|
||||
f"The provided scheduler class ({settings['SCHEDULER']})"
|
||||
" does not fully implement the scheduler interface"
|
||||
)
|
||||
return scheduler_cls
|
||||
|
||||
@inlineCallbacks
|
||||
def start(self) -> Deferred:
|
||||
if self.running:
|
||||
raise RuntimeError("Engine already running")
|
||||
self.start_time = time()
|
||||
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
|
||||
self.running = True
|
||||
self._closewait = defer.Deferred()
|
||||
self._closewait = Deferred()
|
||||
yield self._closewait
|
||||
|
||||
def stop(self):
|
||||
"""Stop the execution engine gracefully"""
|
||||
def stop(self) -> Deferred:
|
||||
"""Gracefully stop the execution engine"""
|
||||
@inlineCallbacks
|
||||
def _finish_stopping_engine(_) -> Deferred:
|
||||
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
|
||||
self._closewait.callback(None)
|
||||
|
||||
if not self.running:
|
||||
raise RuntimeError("Engine not running")
|
||||
|
||||
self.running = False
|
||||
dfd = self._close_all_spiders()
|
||||
return dfd.addBoth(lambda _: self._finish_stopping_engine())
|
||||
dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
|
||||
return dfd.addBoth(_finish_stopping_engine)
|
||||
|
||||
def close(self):
|
||||
"""Close the execution engine gracefully.
|
||||
|
||||
If it has already been started, stop it. In all cases, close all spiders
|
||||
and the downloader.
|
||||
def close(self) -> Deferred:
|
||||
"""
|
||||
Gracefully close the execution engine.
|
||||
If it has already been started, stop it. In all cases, close the spider and the downloader.
|
||||
"""
|
||||
if self.running:
|
||||
# Will also close spiders and downloader
|
||||
return self.stop()
|
||||
elif self.open_spiders:
|
||||
# Will also close downloader
|
||||
return self._close_all_spiders()
|
||||
else:
|
||||
return defer.succeed(self.downloader.close())
|
||||
return self.stop() # will also close spider and downloader
|
||||
if self.spider is not None:
|
||||
return self.close_spider(self.spider, reason="shutdown") # will also close downloader
|
||||
return succeed(self.downloader.close())
|
||||
|
||||
def pause(self):
|
||||
"""Pause the execution engine"""
|
||||
def pause(self) -> None:
|
||||
self.paused = True
|
||||
|
||||
def unpause(self):
|
||||
"""Resume the execution engine"""
|
||||
def unpause(self) -> None:
|
||||
self.paused = False
|
||||
|
||||
def _next_request(self, spider):
|
||||
slot = self.slot
|
||||
if not slot:
|
||||
return
|
||||
def _next_request(self) -> None:
|
||||
assert self.slot is not None # typing
|
||||
assert self.spider is not None # typing
|
||||
|
||||
if self.paused:
|
||||
return
|
||||
return None
|
||||
|
||||
while not self._needs_backout(spider):
|
||||
if not self._next_request_from_scheduler(spider):
|
||||
break
|
||||
while not self._needs_backout() and self._next_request_from_scheduler() is not None:
|
||||
pass
|
||||
|
||||
if slot.start_requests and not self._needs_backout(spider):
|
||||
if self.slot.start_requests is not None and not self._needs_backout():
|
||||
try:
|
||||
request = next(slot.start_requests)
|
||||
request = next(self.slot.start_requests)
|
||||
except StopIteration:
|
||||
slot.start_requests = None
|
||||
self.slot.start_requests = None
|
||||
except Exception:
|
||||
slot.start_requests = None
|
||||
logger.error('Error while obtaining start requests',
|
||||
exc_info=True, extra={'spider': spider})
|
||||
self.slot.start_requests = None
|
||||
logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
|
||||
else:
|
||||
self.crawl(request, spider)
|
||||
self.crawl(request)
|
||||
|
||||
if self.spider_is_idle(spider) and slot.close_if_idle:
|
||||
self._spider_idle(spider)
|
||||
if self.spider_is_idle() and self.slot.close_if_idle:
|
||||
self._spider_idle()
|
||||
|
||||
def _needs_backout(self, spider):
|
||||
slot = self.slot
|
||||
def _needs_backout(self) -> bool:
|
||||
return (
|
||||
not self.running
|
||||
or slot.closing
|
||||
or self.slot.closing # type: ignore[union-attr]
|
||||
or self.downloader.needs_backout()
|
||||
or self.scraper.slot.needs_backout()
|
||||
or self.scraper.slot.needs_backout() # type: ignore[union-attr]
|
||||
)
|
||||
|
||||
def _next_request_from_scheduler(self, spider):
|
||||
slot = self.slot
|
||||
request = slot.scheduler.next_request()
|
||||
if not request:
|
||||
return
|
||||
d = self._download(request, spider)
|
||||
d.addBoth(self._handle_downloader_output, request, spider)
|
||||
def _next_request_from_scheduler(self) -> Optional[Deferred]:
|
||||
assert self.slot is not None # typing
|
||||
assert self.spider is not None # typing
|
||||
|
||||
request = self.slot.scheduler.next_request()
|
||||
if request is None:
|
||||
return None
|
||||
|
||||
d = self._download(request, self.spider)
|
||||
d.addBoth(self._handle_downloader_output, request)
|
||||
d.addErrback(lambda f: logger.info('Error while handling downloader output',
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
d.addBoth(lambda _: slot.remove_request(request))
|
||||
extra={'spider': self.spider}))
|
||||
d.addBoth(lambda _: self.slot.remove_request(request))
|
||||
d.addErrback(lambda f: logger.info('Error while removing request from slot',
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
d.addBoth(lambda _: slot.nextcall.schedule())
|
||||
extra={'spider': self.spider}))
|
||||
d.addBoth(lambda _: self.slot.nextcall.schedule())
|
||||
d.addErrback(lambda f: logger.info('Error while scheduling new request',
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
extra={'spider': self.spider}))
|
||||
return d
|
||||
|
||||
def _handle_downloader_output(self, response, request, spider):
|
||||
if not isinstance(response, (Request, Response, Failure)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Request, Response or Failure, got "
|
||||
f"{type(response)}: {response!r}"
|
||||
)
|
||||
def _handle_downloader_output(
|
||||
self, result: Union[Request, Response, Failure], request: Request
|
||||
) -> Optional[Deferred]:
|
||||
assert self.spider is not None # typing
|
||||
|
||||
if not isinstance(result, (Request, Response, Failure)):
|
||||
raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
|
||||
|
||||
# downloader middleware can return requests (for example, redirects)
|
||||
if isinstance(response, Request):
|
||||
self.crawl(response, spider)
|
||||
return
|
||||
# response is a Response or Failure
|
||||
d = self.scraper.enqueue_scrape(response, request, spider)
|
||||
d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
if isinstance(result, Request):
|
||||
self.crawl(result)
|
||||
return None
|
||||
|
||||
d = self.scraper.enqueue_scrape(result, request, self.spider)
|
||||
d.addErrback(
|
||||
lambda f: logger.error(
|
||||
"Error while enqueuing downloader output",
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': self.spider},
|
||||
)
|
||||
)
|
||||
return d
|
||||
|
||||
def spider_is_idle(self, spider):
|
||||
if not self.scraper.slot.is_idle():
|
||||
# scraper is not idle
|
||||
def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
|
||||
if spider is not None:
|
||||
warnings.warn(
|
||||
"Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if self.slot is None:
|
||||
raise RuntimeError("Engine slot not assigned")
|
||||
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
|
||||
return False
|
||||
|
||||
if self.downloader.active:
|
||||
# downloader has pending requests
|
||||
if self.downloader.active: # downloader has pending requests
|
||||
return False
|
||||
|
||||
if self.slot.start_requests is not None:
|
||||
# not all start requests are handled
|
||||
if self.slot.start_requests is not None: # not all start requests are handled
|
||||
return False
|
||||
|
||||
if self.slot.scheduler.has_pending_requests():
|
||||
# scheduler has pending requests
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@property
|
||||
def open_spiders(self):
|
||||
return [self.spider] if self.spider else []
|
||||
def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
|
||||
"""Inject the request into the spider <-> downloader pipeline"""
|
||||
if spider is not None:
|
||||
warnings.warn(
|
||||
"Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if spider is not self.spider:
|
||||
raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
|
||||
if self.spider is None:
|
||||
raise RuntimeError(f"No open spider to crawl: {request}")
|
||||
self._schedule_request(request, self.spider)
|
||||
self.slot.nextcall.schedule() # type: ignore[union-attr]
|
||||
|
||||
def has_capacity(self):
|
||||
"""Does the engine have capacity to handle more spiders"""
|
||||
return not bool(self.slot)
|
||||
|
||||
def crawl(self, request, spider):
|
||||
if spider not in self.open_spiders:
|
||||
raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}")
|
||||
self.schedule(request, spider)
|
||||
self.slot.nextcall.schedule()
|
||||
|
||||
def schedule(self, request, spider):
|
||||
def _schedule_request(self, request: Request, spider: Spider) -> None:
|
||||
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
|
||||
if not self.slot.scheduler.enqueue_request(request):
|
||||
if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
|
||||
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
|
||||
|
||||
def download(self, request, spider):
|
||||
d = self._download(request, spider)
|
||||
d.addBoth(self._downloaded, self.slot, request, spider)
|
||||
return d
|
||||
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
|
||||
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
|
||||
if spider is None:
|
||||
spider = self.spider
|
||||
else:
|
||||
warnings.warn(
|
||||
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if spider is not self.spider:
|
||||
logger.warning("The spider '%s' does not match the open spider", spider.name)
|
||||
if spider is None:
|
||||
raise RuntimeError(f"No open spider to crawl: {request}")
|
||||
return self._download(request, spider).addBoth(self._downloaded, request, spider)
|
||||
|
||||
def _downloaded(self, response, slot, request, spider):
|
||||
slot.remove_request(request)
|
||||
return self.download(response, spider) if isinstance(response, Request) else response
|
||||
def _downloaded(
|
||||
self, result: Union[Response, Request], request: Request, spider: Spider
|
||||
) -> Union[Deferred, Response]:
|
||||
assert self.slot is not None # typing
|
||||
self.slot.remove_request(request)
|
||||
return self.download(result, spider) if isinstance(result, Request) else result
|
||||
|
||||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
slot.add_request(request)
|
||||
def _download(self, request: Request, spider: Spider) -> Deferred:
|
||||
assert self.slot is not None # typing
|
||||
|
||||
def _on_success(response):
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Response or Request, got "
|
||||
f"{type(response)}: {response!r}"
|
||||
)
|
||||
if isinstance(response, Response):
|
||||
if response.request is None:
|
||||
response.request = request
|
||||
logkws = self.logformatter.crawled(response.request, response, spider)
|
||||
self.slot.add_request(request)
|
||||
|
||||
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
|
||||
if not isinstance(result, (Response, Request)):
|
||||
raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
|
||||
if isinstance(result, Response):
|
||||
if result.request is None:
|
||||
result.request = request
|
||||
logkws = self.logformatter.crawled(result.request, result, spider)
|
||||
if logkws is not None:
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
|
||||
logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
|
||||
self.signals.send_catch_log(
|
||||
signal=signals.response_received,
|
||||
response=response,
|
||||
request=response.request,
|
||||
response=result,
|
||||
request=result.request,
|
||||
spider=spider,
|
||||
)
|
||||
return response
|
||||
return result
|
||||
|
||||
def _on_complete(_):
|
||||
slot.nextcall.schedule()
|
||||
self.slot.nextcall.schedule()
|
||||
return _
|
||||
|
||||
dwld = self.downloader.fetch(request, spider)
|
||||
|
|
@ -265,58 +306,62 @@ class ExecutionEngine:
|
|||
dwld.addBoth(_on_complete)
|
||||
return dwld
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def open_spider(self, spider, start_requests=(), close_if_idle=True):
|
||||
if not self.has_capacity():
|
||||
@inlineCallbacks
|
||||
def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
|
||||
if self.slot is not None:
|
||||
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
|
||||
logger.info("Spider opened", extra={'spider': spider})
|
||||
nextcall = CallLaterOnce(self._next_request, spider)
|
||||
scheduler = self.scheduler_cls.from_crawler(self.crawler)
|
||||
nextcall = CallLaterOnce(self._next_request)
|
||||
scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
|
||||
start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
|
||||
slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
|
||||
self.slot = slot
|
||||
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
|
||||
self.spider = spider
|
||||
yield scheduler.open(spider)
|
||||
if hasattr(scheduler, "open"):
|
||||
yield scheduler.open(spider)
|
||||
yield self.scraper.open_spider(spider)
|
||||
self.crawler.stats.open_spider(spider)
|
||||
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
|
||||
slot.nextcall.schedule()
|
||||
slot.heartbeat.start(5)
|
||||
self.slot.nextcall.schedule()
|
||||
self.slot.heartbeat.start(5)
|
||||
|
||||
def _spider_idle(self, spider):
|
||||
"""Called when a spider gets idle. This function is called when there
|
||||
are no remaining pages to download or schedule. It can be called
|
||||
multiple times. If some extension raises a DontCloseSpider exception
|
||||
(in the spider_idle signal handler) the spider is not closed until the
|
||||
next loop and this function is guaranteed to be called (at least) once
|
||||
again for this spider.
|
||||
def _spider_idle(self) -> None:
|
||||
"""
|
||||
res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
|
||||
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
|
||||
return
|
||||
Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
|
||||
It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
|
||||
exception, the spider is not closed until the next loop and this function is guaranteed to be called
|
||||
(at least) once again. A handler can raise CloseSpider to provide a custom closing reason.
|
||||
"""
|
||||
assert self.spider is not None # typing
|
||||
expected_ex = (DontCloseSpider, CloseSpider)
|
||||
res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
|
||||
detected_ex = {
|
||||
ex: x.value
|
||||
for _, x in res
|
||||
for ex in expected_ex
|
||||
if isinstance(x, Failure) and isinstance(x.value, ex)
|
||||
}
|
||||
if DontCloseSpider in detected_ex:
|
||||
return None
|
||||
if self.spider_is_idle():
|
||||
ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished'))
|
||||
assert isinstance(ex, CloseSpider) # typing
|
||||
self.close_spider(self.spider, reason=ex.reason)
|
||||
|
||||
if self.spider_is_idle(spider):
|
||||
self.close_spider(spider, reason='finished')
|
||||
|
||||
def close_spider(self, spider, reason='cancelled'):
|
||||
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
|
||||
"""Close (cancel) spider and clear all its outstanding requests"""
|
||||
if self.slot is None:
|
||||
raise RuntimeError("Engine slot not assigned")
|
||||
|
||||
slot = self.slot
|
||||
if slot.closing:
|
||||
return slot.closing
|
||||
logger.info("Closing spider (%(reason)s)",
|
||||
{'reason': reason},
|
||||
extra={'spider': spider})
|
||||
if self.slot.closing is not None:
|
||||
return self.slot.closing
|
||||
|
||||
dfd = slot.close()
|
||||
logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
|
||||
|
||||
def log_failure(msg):
|
||||
def errback(failure):
|
||||
logger.error(
|
||||
msg,
|
||||
exc_info=failure_to_exc_info(failure),
|
||||
extra={'spider': spider}
|
||||
)
|
||||
dfd = self.slot.close()
|
||||
|
||||
def log_failure(msg: str) -> Callable:
|
||||
def errback(failure: Failure) -> None:
|
||||
logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
|
||||
return errback
|
||||
|
||||
dfd.addBoth(lambda _: self.downloader.close())
|
||||
|
|
@ -325,19 +370,19 @@ class ExecutionEngine:
|
|||
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
|
||||
dfd.addErrback(log_failure('Scraper close failure'))
|
||||
|
||||
dfd.addBoth(lambda _: slot.scheduler.close(reason))
|
||||
dfd.addErrback(log_failure('Scheduler close failure'))
|
||||
if hasattr(self.slot.scheduler, "close"):
|
||||
dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
|
||||
dfd.addErrback(log_failure("Scheduler close failure"))
|
||||
|
||||
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
|
||||
signal=signals.spider_closed, spider=spider, reason=reason))
|
||||
signal=signals.spider_closed, spider=spider, reason=reason,
|
||||
))
|
||||
dfd.addErrback(log_failure('Error while sending spider_close signal'))
|
||||
|
||||
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
|
||||
dfd.addErrback(log_failure('Stats close failure'))
|
||||
|
||||
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
|
||||
{'reason': reason},
|
||||
extra={'spider': spider}))
|
||||
dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
|
||||
|
||||
dfd.addBoth(lambda _: setattr(self, 'slot', None))
|
||||
dfd.addErrback(log_failure('Error while unassigning slot'))
|
||||
|
|
@ -349,12 +394,26 @@ class ExecutionEngine:
|
|||
|
||||
return dfd
|
||||
|
||||
def _close_all_spiders(self):
|
||||
dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
|
||||
dlist = defer.DeferredList(dfds)
|
||||
return dlist
|
||||
@property
|
||||
def open_spiders(self) -> list:
|
||||
warnings.warn(
|
||||
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return [self.spider] if self.spider is not None else []
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _finish_stopping_engine(self):
|
||||
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
|
||||
self._closewait.callback(None)
|
||||
def has_capacity(self) -> bool:
|
||||
warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
|
||||
return not bool(self.slot)
|
||||
|
||||
def schedule(self, request: Request, spider: Spider) -> None:
|
||||
warnings.warn(
|
||||
"ExecutionEngine.schedule is deprecated, please use "
|
||||
"ExecutionEngine.crawl or ExecutionEngine.download instead",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if self.slot is None:
|
||||
raise RuntimeError("Engine slot not assigned")
|
||||
self._schedule_request(request, spider)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
from collections import deque
|
||||
from typing import Deque, Dict, List, Optional, Tuple
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.base import ReactorBase
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.endpoints import HostnameEndpoint
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
|
||||
from twisted.web.error import SchemeNotSupported
|
||||
|
||||
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
|
||||
class H2ConnectionPool:
|
||||
def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
|
||||
self._reactor = reactor
|
||||
self.settings = settings
|
||||
|
||||
# Store a dictionary which is used to get the respective
|
||||
# H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port)
|
||||
self._connections: Dict[Tuple, H2ClientProtocol] = {}
|
||||
|
||||
# Save all requests that arrive before the connection is established
|
||||
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
|
||||
|
||||
def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
|
||||
if key in self._pending_requests:
|
||||
# Received a request while connecting to remote
|
||||
# Create a deferred which will fire with the H2ClientProtocol
|
||||
# instance
|
||||
d = Deferred()
|
||||
self._pending_requests[key].append(d)
|
||||
return d
|
||||
|
||||
# Check if we already have a connection to the remote
|
||||
conn = self._connections.get(key, None)
|
||||
if conn:
|
||||
# Return this connection instance wrapped inside a deferred
|
||||
return defer.succeed(conn)
|
||||
|
||||
# No connection is established for the given URI
|
||||
return self._new_connection(key, uri, endpoint)
|
||||
|
||||
def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
|
||||
self._pending_requests[key] = deque()
|
||||
|
||||
conn_lost_deferred = Deferred()
|
||||
conn_lost_deferred.addCallback(self._remove_connection, key)
|
||||
|
||||
factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
|
||||
conn_d = endpoint.connect(factory)
|
||||
conn_d.addCallback(self.put_connection, key)
|
||||
|
||||
d = Deferred()
|
||||
self._pending_requests[key].append(d)
|
||||
return d
|
||||
|
||||
def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
|
||||
self._connections[key] = conn
|
||||
|
||||
# Now as we have established a proper HTTP/2 connection
|
||||
# we fire all the deferred's with the connection instance
|
||||
pending_requests = self._pending_requests.pop(key, None)
|
||||
while pending_requests:
|
||||
d = pending_requests.popleft()
|
||||
d.callback(conn)
|
||||
|
||||
return conn
|
||||
|
||||
def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
|
||||
self._connections.pop(key)
|
||||
|
||||
# Call the errback of all the pending requests for this connection
|
||||
pending_requests = self._pending_requests.pop(key, None)
|
||||
while pending_requests:
|
||||
d = pending_requests.popleft()
|
||||
d.errback(errors)
|
||||
|
||||
def close_connections(self) -> None:
|
||||
"""Close all the HTTP/2 connections and remove them from pool
|
||||
|
||||
Returns:
|
||||
Deferred that fires when all connections have been closed
|
||||
"""
|
||||
for conn in self._connections.values():
|
||||
conn.transport.abortConnection()
|
||||
|
||||
|
||||
class H2Agent:
|
||||
def __init__(
|
||||
self,
|
||||
reactor: ReactorBase,
|
||||
pool: H2ConnectionPool,
|
||||
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
|
||||
connect_timeout: Optional[float] = None,
|
||||
bind_address: Optional[bytes] = None,
|
||||
) -> None:
|
||||
self._reactor = reactor
|
||||
self._pool = pool
|
||||
self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
|
||||
self.endpoint_factory = _StandardEndpointFactory(
|
||||
self._reactor, self._context_factory, connect_timeout, bind_address
|
||||
)
|
||||
|
||||
def get_endpoint(self, uri: URI):
|
||||
return self.endpoint_factory.endpointForURI(uri)
|
||||
|
||||
def get_key(self, uri: URI) -> Tuple:
|
||||
"""
|
||||
Arguments:
|
||||
uri - URI obtained directly from request URL
|
||||
"""
|
||||
return uri.scheme, uri.host, uri.port
|
||||
|
||||
def request(self, request: Request, spider: Spider) -> Deferred:
|
||||
uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
|
||||
try:
|
||||
endpoint = self.get_endpoint(uri)
|
||||
except SchemeNotSupported:
|
||||
return defer.fail(Failure())
|
||||
|
||||
key = self.get_key(uri)
|
||||
d = self._pool.get_connection(key, uri, endpoint)
|
||||
d.addCallback(lambda conn: conn.request(request, spider))
|
||||
return d
|
||||
|
||||
|
||||
class ScrapyProxyH2Agent(H2Agent):
|
||||
def __init__(
|
||||
self,
|
||||
reactor: ReactorBase,
|
||||
proxy_uri: URI,
|
||||
pool: H2ConnectionPool,
|
||||
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
|
||||
connect_timeout: Optional[float] = None,
|
||||
bind_address: Optional[bytes] = None,
|
||||
) -> None:
|
||||
super(ScrapyProxyH2Agent, self).__init__(
|
||||
reactor=reactor,
|
||||
pool=pool,
|
||||
context_factory=context_factory,
|
||||
connect_timeout=connect_timeout,
|
||||
bind_address=bind_address,
|
||||
)
|
||||
self._proxy_uri = proxy_uri
|
||||
|
||||
def get_endpoint(self, uri: URI):
|
||||
return self.endpoint_factory.endpointForURI(self._proxy_uri)
|
||||
|
||||
def get_key(self, uri: URI) -> Tuple:
|
||||
"""We use the proxy uri instead of uri obtained from request url"""
|
||||
return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
|
||||
|
|
@ -0,0 +1,418 @@
|
|||
import ipaddress
|
||||
import itertools
|
||||
import logging
|
||||
from collections import deque
|
||||
from ipaddress import IPv4Address, IPv6Address
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
from h2.config import H2Configuration
|
||||
from h2.connection import H2Connection
|
||||
from h2.errors import ErrorCodes
|
||||
from h2.events import (
|
||||
Event, ConnectionTerminated, DataReceived, ResponseReceived,
|
||||
SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
|
||||
WindowUpdated
|
||||
)
|
||||
from h2.exceptions import FrameTooLargeError, H2Error
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
|
||||
from twisted.internet.protocol import connectionDone, Factory, Protocol
|
||||
from twisted.internet.ssl import Certificate
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.web.client import URI
|
||||
from zope.interface import implementer
|
||||
|
||||
from scrapy.core.http2.stream import Stream, StreamCloseReason
|
||||
from scrapy.http import Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
PROTOCOL_NAME = b"h2"
|
||||
|
||||
|
||||
class InvalidNegotiatedProtocol(H2Error):
|
||||
|
||||
def __init__(self, negotiated_protocol: bytes) -> None:
|
||||
self.negotiated_protocol = negotiated_protocol
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
|
||||
|
||||
|
||||
class RemoteTerminatedConnection(H2Error):
|
||||
def __init__(
|
||||
self,
|
||||
remote_ip_address: Optional[Union[IPv4Address, IPv6Address]],
|
||||
event: ConnectionTerminated,
|
||||
) -> None:
|
||||
self.remote_ip_address = remote_ip_address
|
||||
self.terminate_event = event
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'Received GOAWAY frame from {self.remote_ip_address!r}'
|
||||
|
||||
|
||||
class MethodNotAllowed405(H2Error):
|
||||
def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
|
||||
self.remote_ip_address = remote_ip_address
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
|
||||
|
||||
|
||||
@implementer(IHandshakeListener)
|
||||
class H2ClientProtocol(Protocol, TimeoutMixin):
|
||||
IDLE_TIMEOUT = 240
|
||||
|
||||
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
|
||||
"""
|
||||
Arguments:
|
||||
uri -- URI of the base url to which HTTP/2 Connection will be made.
|
||||
uri is used to verify that incoming client requests have correct
|
||||
base URL.
|
||||
settings -- Scrapy project settings
|
||||
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
|
||||
that connection was lost
|
||||
"""
|
||||
self._conn_lost_deferred = conn_lost_deferred
|
||||
|
||||
config = H2Configuration(client_side=True, header_encoding='utf-8')
|
||||
self.conn = H2Connection(config=config)
|
||||
|
||||
# ID of the next request stream
|
||||
# Following the convention - 'Streams initiated by a client MUST
|
||||
# use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1)
|
||||
self._stream_id_generator = itertools.count(start=1, step=2)
|
||||
|
||||
# Streams are stored in a dictionary keyed off their stream IDs
|
||||
self.streams: Dict[int, Stream] = {}
|
||||
|
||||
# If requests are received before connection is made we keep
|
||||
# all requests in a pool and send them as the connection is made
|
||||
self._pending_request_stream_pool: deque = deque()
|
||||
|
||||
# Save an instance of errors raised which lead to losing the connection
|
||||
# We pass these instances to the streams ResponseFailed() failure
|
||||
self._conn_lost_errors: List[BaseException] = []
|
||||
|
||||
# Some meta data of this connection
|
||||
# initialized when connection is successfully made
|
||||
self.metadata: Dict = {
|
||||
# Peer certificate instance
|
||||
'certificate': None,
|
||||
|
||||
# Address of the server we are connected to which
|
||||
# is updated when HTTP/2 connection is made successfully
|
||||
'ip_address': None,
|
||||
|
||||
# URI of the peer HTTP/2 connection is made
|
||||
'uri': uri,
|
||||
|
||||
# Both ip_address and uri are used by the Stream before
|
||||
# initiating the request to verify that the base address
|
||||
|
||||
# Variables taken from Project Settings
|
||||
'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
|
||||
'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
|
||||
|
||||
# Counter to keep track of opened streams. This counter
|
||||
# is used to make sure that not more than MAX_CONCURRENT_STREAMS
|
||||
# streams are opened which leads to ProtocolError
|
||||
# We use simple FIFO policy to handle pending requests
|
||||
'active_streams': 0,
|
||||
|
||||
# Flag to keep track if settings were acknowledged by the remote
|
||||
# This ensures that we have established a HTTP/2 connection
|
||||
'settings_acknowledged': False,
|
||||
}
|
||||
|
||||
@property
|
||||
def h2_connected(self) -> bool:
|
||||
"""Boolean to keep track of the connection status.
|
||||
This is used while initiating pending streams to make sure
|
||||
that we initiate stream only during active HTTP/2 Connection
|
||||
"""
|
||||
return bool(self.transport.connected) and self.metadata['settings_acknowledged']
|
||||
|
||||
@property
|
||||
def allowed_max_concurrent_streams(self) -> int:
|
||||
"""We keep total two streams for client (sending data) and
|
||||
server side (receiving data) for a single request. To be safe
|
||||
we choose the minimum. Since this value can change in event
|
||||
RemoteSettingsChanged we make variable a property.
|
||||
"""
|
||||
return min(
|
||||
self.conn.local_settings.max_concurrent_streams,
|
||||
self.conn.remote_settings.max_concurrent_streams
|
||||
)
|
||||
|
||||
def _send_pending_requests(self) -> None:
|
||||
"""Initiate all pending requests from the deque following FIFO
|
||||
We make sure that at any time {allowed_max_concurrent_streams}
|
||||
streams are active.
|
||||
"""
|
||||
while (
|
||||
self._pending_request_stream_pool
|
||||
and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
|
||||
and self.h2_connected
|
||||
):
|
||||
self.metadata['active_streams'] += 1
|
||||
stream = self._pending_request_stream_pool.popleft()
|
||||
stream.initiate_request()
|
||||
self._write_to_transport()
|
||||
|
||||
def pop_stream(self, stream_id: int) -> Stream:
|
||||
"""Perform cleanup when a stream is closed
|
||||
"""
|
||||
stream = self.streams.pop(stream_id)
|
||||
self.metadata['active_streams'] -= 1
|
||||
self._send_pending_requests()
|
||||
return stream
|
||||
|
||||
def _new_stream(self, request: Request, spider: Spider) -> Stream:
|
||||
"""Instantiates a new Stream object
|
||||
"""
|
||||
stream = Stream(
|
||||
stream_id=next(self._stream_id_generator),
|
||||
request=request,
|
||||
protocol=self,
|
||||
download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
|
||||
download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
|
||||
)
|
||||
self.streams[stream.stream_id] = stream
|
||||
return stream
|
||||
|
||||
def _write_to_transport(self) -> None:
|
||||
""" Write data to the underlying transport connection
|
||||
from the HTTP2 connection instance if any
|
||||
"""
|
||||
# Reset the idle timeout as connection is still actively sending data
|
||||
self.resetTimeout()
|
||||
|
||||
data = self.conn.data_to_send()
|
||||
self.transport.write(data)
|
||||
|
||||
def request(self, request: Request, spider: Spider) -> Deferred:
|
||||
if not isinstance(request, Request):
|
||||
raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
|
||||
|
||||
stream = self._new_stream(request, spider)
|
||||
d = stream.get_response()
|
||||
|
||||
# Add the stream to the request pool
|
||||
self._pending_request_stream_pool.append(stream)
|
||||
|
||||
# If we receive a request when connection is idle
|
||||
# We need to initiate pending requests
|
||||
self._send_pending_requests()
|
||||
return d
|
||||
|
||||
def connectionMade(self) -> None:
|
||||
"""Called by Twisted when the connection is established. We can start
|
||||
sending some data now: we should open with the connection preamble.
|
||||
"""
|
||||
# Initialize the timeout
|
||||
self.setTimeout(self.IDLE_TIMEOUT)
|
||||
|
||||
destination = self.transport.getPeer()
|
||||
self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
|
||||
|
||||
# Initiate H2 Connection
|
||||
self.conn.initiate_connection()
|
||||
self._write_to_transport()
|
||||
|
||||
def _lose_connection_with_error(self, errors: List[BaseException]) -> None:
|
||||
"""Helper function to lose the connection with the error sent as a
|
||||
reason"""
|
||||
self._conn_lost_errors += errors
|
||||
self.transport.loseConnection()
|
||||
|
||||
def handshakeCompleted(self) -> None:
|
||||
"""
|
||||
Close the connection if it's not made via the expected protocol
|
||||
"""
|
||||
if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
|
||||
# we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
|
||||
self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
|
||||
|
||||
def _check_received_data(self, data: bytes) -> None:
|
||||
"""Checks for edge cases where the connection to remote fails
|
||||
without raising an appropriate H2Error
|
||||
|
||||
Arguments:
|
||||
data -- Data received from the remote
|
||||
"""
|
||||
if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
|
||||
raise MethodNotAllowed405(self.metadata['ip_address'])
|
||||
|
||||
def dataReceived(self, data: bytes) -> None:
|
||||
# Reset the idle timeout as connection is still actively receiving data
|
||||
self.resetTimeout()
|
||||
|
||||
try:
|
||||
self._check_received_data(data)
|
||||
events = self.conn.receive_data(data)
|
||||
self._handle_events(events)
|
||||
except H2Error as e:
|
||||
if isinstance(e, FrameTooLargeError):
|
||||
# hyper-h2 does not drop the connection in this scenario, we
|
||||
# need to abort the connection manually.
|
||||
self._conn_lost_errors += [e]
|
||||
self.transport.abortConnection()
|
||||
return
|
||||
|
||||
# Save this error as ultimately the connection will be dropped
|
||||
# internally by hyper-h2. Saved error will be passed to all the streams
|
||||
# closed with the connection.
|
||||
self._lose_connection_with_error([e])
|
||||
finally:
|
||||
self._write_to_transport()
|
||||
|
||||
def timeoutConnection(self) -> None:
|
||||
"""Called when the connection times out.
|
||||
We lose the connection with TimeoutError"""
|
||||
|
||||
# Check whether there are open streams. If there are, we're going to
|
||||
# want to use the error code PROTOCOL_ERROR. If there aren't, use
|
||||
# NO_ERROR.
|
||||
if (
|
||||
self.conn.open_outbound_streams > 0
|
||||
or self.conn.open_inbound_streams > 0
|
||||
or self.metadata['active_streams'] > 0
|
||||
):
|
||||
error_code = ErrorCodes.PROTOCOL_ERROR
|
||||
else:
|
||||
error_code = ErrorCodes.NO_ERROR
|
||||
self.conn.close_connection(error_code=error_code)
|
||||
self._write_to_transport()
|
||||
|
||||
self._lose_connection_with_error([
|
||||
TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
|
||||
])
|
||||
|
||||
def connectionLost(self, reason: Failure = connectionDone) -> None:
|
||||
"""Called by Twisted when the transport connection is lost.
|
||||
No need to write anything to transport here.
|
||||
"""
|
||||
# Cancel the timeout if not done yet
|
||||
self.setTimeout(None)
|
||||
|
||||
# Notify the connection pool instance such that no new requests are
|
||||
# sent over current connection
|
||||
if not reason.check(connectionDone):
|
||||
self._conn_lost_errors.append(reason)
|
||||
|
||||
self._conn_lost_deferred.callback(self._conn_lost_errors)
|
||||
|
||||
for stream in self.streams.values():
|
||||
if stream.metadata['request_sent']:
|
||||
close_reason = StreamCloseReason.CONNECTION_LOST
|
||||
else:
|
||||
close_reason = StreamCloseReason.INACTIVE
|
||||
stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
|
||||
|
||||
self.metadata['active_streams'] -= len(self.streams)
|
||||
self.streams.clear()
|
||||
self._pending_request_stream_pool.clear()
|
||||
self.conn.close_connection()
|
||||
|
||||
def _handle_events(self, events: List[Event]) -> None:
|
||||
"""Private method which acts as a bridge between the events
|
||||
received from the HTTP/2 data and IH2EventsHandler
|
||||
|
||||
Arguments:
|
||||
events -- A list of events that the remote peer triggered by sending data
|
||||
"""
|
||||
for event in events:
|
||||
if isinstance(event, ConnectionTerminated):
|
||||
self.connection_terminated(event)
|
||||
elif isinstance(event, DataReceived):
|
||||
self.data_received(event)
|
||||
elif isinstance(event, ResponseReceived):
|
||||
self.response_received(event)
|
||||
elif isinstance(event, StreamEnded):
|
||||
self.stream_ended(event)
|
||||
elif isinstance(event, StreamReset):
|
||||
self.stream_reset(event)
|
||||
elif isinstance(event, WindowUpdated):
|
||||
self.window_updated(event)
|
||||
elif isinstance(event, SettingsAcknowledged):
|
||||
self.settings_acknowledged(event)
|
||||
elif isinstance(event, UnknownFrameReceived):
|
||||
logger.warning('Unknown frame received: %s', event.frame)
|
||||
|
||||
# Event handler functions starts here
|
||||
def connection_terminated(self, event: ConnectionTerminated) -> None:
|
||||
self._lose_connection_with_error([
|
||||
RemoteTerminatedConnection(self.metadata['ip_address'], event)
|
||||
])
|
||||
|
||||
def data_received(self, event: DataReceived) -> None:
|
||||
try:
|
||||
stream = self.streams[event.stream_id]
|
||||
except KeyError:
|
||||
pass # We ignore server-initiated events
|
||||
else:
|
||||
stream.receive_data(event.data, event.flow_controlled_length)
|
||||
|
||||
def response_received(self, event: ResponseReceived) -> None:
|
||||
try:
|
||||
stream = self.streams[event.stream_id]
|
||||
except KeyError:
|
||||
pass # We ignore server-initiated events
|
||||
else:
|
||||
stream.receive_headers(event.headers)
|
||||
|
||||
def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
|
||||
self.metadata['settings_acknowledged'] = True
|
||||
|
||||
# Send off all the pending requests as now we have
|
||||
# established a proper HTTP/2 connection
|
||||
self._send_pending_requests()
|
||||
|
||||
# Update certificate when our HTTP/2 connection is established
|
||||
self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
|
||||
|
||||
def stream_ended(self, event: StreamEnded) -> None:
|
||||
try:
|
||||
stream = self.pop_stream(event.stream_id)
|
||||
except KeyError:
|
||||
pass # We ignore server-initiated events
|
||||
else:
|
||||
stream.close(StreamCloseReason.ENDED, from_protocol=True)
|
||||
|
||||
def stream_reset(self, event: StreamReset) -> None:
|
||||
try:
|
||||
stream = self.pop_stream(event.stream_id)
|
||||
except KeyError:
|
||||
pass # We ignore server-initiated events
|
||||
else:
|
||||
stream.close(StreamCloseReason.RESET, from_protocol=True)
|
||||
|
||||
def window_updated(self, event: WindowUpdated) -> None:
|
||||
if event.stream_id != 0:
|
||||
self.streams[event.stream_id].receive_window_update()
|
||||
else:
|
||||
# Send leftover data for all the streams
|
||||
for stream in self.streams.values():
|
||||
stream.receive_window_update()
|
||||
|
||||
|
||||
@implementer(IProtocolNegotiationFactory)
|
||||
class H2ClientFactory(Factory):
|
||||
def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
|
||||
self.uri = uri
|
||||
self.settings = settings
|
||||
self.conn_lost_deferred = conn_lost_deferred
|
||||
|
||||
def buildProtocol(self, addr) -> H2ClientProtocol:
|
||||
return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
|
||||
|
||||
def acceptableProtocols(self) -> List[bytes]:
|
||||
return [PROTOCOL_NAME]
|
||||
|
|
@ -0,0 +1,470 @@
|
|||
import logging
|
||||
from enum import Enum
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
|
||||
|
||||
from h2.errors import ErrorCodes
|
||||
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
|
||||
from hpack import HeaderTuple
|
||||
from twisted.internet.defer import Deferred, CancelledError
|
||||
from twisted.internet.error import ConnectionClosed
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.core.http2.protocol import H2ClientProtocol
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InactiveStreamClosed(ConnectionClosed):
|
||||
"""Connection was closed without sending request headers
|
||||
of the stream. This happens when a stream is waiting for other
|
||||
streams to close and connection is lost."""
|
||||
|
||||
def __init__(self, request: Request) -> None:
|
||||
self.request = request
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
|
||||
|
||||
|
||||
class InvalidHostname(H2Error):
|
||||
|
||||
def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
|
||||
self.request = request
|
||||
self.expected_hostname = expected_hostname
|
||||
self.expected_netloc = expected_netloc
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
|
||||
|
||||
|
||||
class StreamCloseReason(Enum):
|
||||
# Received a StreamEnded event from the remote
|
||||
ENDED = 1
|
||||
|
||||
# Received a StreamReset event -- ended abruptly
|
||||
RESET = 2
|
||||
|
||||
# Transport connection was lost
|
||||
CONNECTION_LOST = 3
|
||||
|
||||
# Expected response body size is more than allowed limit
|
||||
MAXSIZE_EXCEEDED = 4
|
||||
|
||||
# Response deferred is cancelled by the client
|
||||
# (happens when client called response_deferred.cancel())
|
||||
CANCELLED = 5
|
||||
|
||||
# Connection lost and the stream was not initiated
|
||||
INACTIVE = 6
|
||||
|
||||
# The hostname of the request is not same as of connected peer hostname
|
||||
# As a result sending this request will the end the connection
|
||||
INVALID_HOSTNAME = 7
|
||||
|
||||
|
||||
class Stream:
|
||||
"""Represents a single HTTP/2 Stream.
|
||||
|
||||
Stream is a bidirectional flow of bytes within an established connection,
|
||||
which may carry one or more messages. Handles the transfer of HTTP Headers
|
||||
and Data frames.
|
||||
|
||||
Role of this class is to
|
||||
1. Combine all the data frames
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stream_id: int,
|
||||
request: Request,
|
||||
protocol: "H2ClientProtocol",
|
||||
download_maxsize: int = 0,
|
||||
download_warnsize: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Arguments:
|
||||
stream_id -- Unique identifier for the stream within a single HTTP/2 connection
|
||||
request -- The HTTP request associated to the stream
|
||||
protocol -- Parent H2ClientProtocol instance
|
||||
"""
|
||||
self.stream_id: int = stream_id
|
||||
self._request: Request = request
|
||||
self._protocol: "H2ClientProtocol" = protocol
|
||||
|
||||
self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
|
||||
self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
|
||||
|
||||
# Metadata of an HTTP/2 connection stream
|
||||
# initialized when stream is instantiated
|
||||
self.metadata: Dict = {
|
||||
'request_content_length': 0 if self._request.body is None else len(self._request.body),
|
||||
|
||||
# Flag to keep track whether the stream has initiated the request
|
||||
'request_sent': False,
|
||||
|
||||
# Flag to track whether we have logged about exceeding download warnsize
|
||||
'reached_warnsize': False,
|
||||
|
||||
# Each time we send a data frame, we will decrease value by the amount send.
|
||||
'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
|
||||
|
||||
# Flag to keep track whether client (self) have closed this stream
|
||||
'stream_closed_local': False,
|
||||
|
||||
# Flag to keep track whether the server has closed the stream
|
||||
'stream_closed_server': False,
|
||||
}
|
||||
|
||||
# Private variable used to build the response
|
||||
# this response is then converted to appropriate Response class
|
||||
# passed to the response deferred callback
|
||||
self._response: Dict = {
|
||||
# Data received frame by frame from the server is appended
|
||||
# and passed to the response Deferred when completely received.
|
||||
'body': BytesIO(),
|
||||
|
||||
# The amount of data received that counts against the
|
||||
# flow control window
|
||||
'flow_controlled_size': 0,
|
||||
|
||||
# Headers received after sending the request
|
||||
'headers': Headers({}),
|
||||
}
|
||||
|
||||
def _cancel(_) -> None:
|
||||
# Close this stream as gracefully as possible
|
||||
# If the associated request is initiated we reset this stream
|
||||
# else we directly call close() method
|
||||
if self.metadata['request_sent']:
|
||||
self.reset_stream(StreamCloseReason.CANCELLED)
|
||||
else:
|
||||
self.close(StreamCloseReason.CANCELLED)
|
||||
|
||||
self._deferred_response = Deferred(_cancel)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f'Stream(id={self.stream_id!r})'
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
@property
|
||||
def _log_warnsize(self) -> bool:
|
||||
"""Checks if we have received data which exceeds the download warnsize
|
||||
and whether we have not already logged about it.
|
||||
|
||||
Returns:
|
||||
True if both the above conditions hold true
|
||||
False if any of the conditions is false
|
||||
"""
|
||||
content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
|
||||
return (
|
||||
self._download_warnsize
|
||||
and (
|
||||
self._response['flow_controlled_size'] > self._download_warnsize
|
||||
or content_length_header > self._download_warnsize
|
||||
)
|
||||
and not self.metadata['reached_warnsize']
|
||||
)
|
||||
|
||||
def get_response(self) -> Deferred:
|
||||
"""Simply return a Deferred which fires when response
|
||||
from the asynchronous request is available
|
||||
"""
|
||||
return self._deferred_response
|
||||
|
||||
def check_request_url(self) -> bool:
|
||||
# Make sure that we are sending the request to the correct URL
|
||||
url = urlparse(self._request.url)
|
||||
return (
|
||||
url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
|
||||
or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
|
||||
or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
|
||||
)
|
||||
|
||||
def _get_request_headers(self) -> List[Tuple[str, str]]:
|
||||
url = urlparse(self._request.url)
|
||||
|
||||
path = url.path
|
||||
if url.query:
|
||||
path += '?' + url.query
|
||||
|
||||
# This pseudo-header field MUST NOT be empty for "http" or "https"
|
||||
# URIs; "http" or "https" URIs that do not contain a path component
|
||||
# MUST include a value of '/'. The exception to this rule is an
|
||||
# OPTIONS request for an "http" or "https" URI that does not include
|
||||
# a path component; these MUST include a ":path" pseudo-header field
|
||||
# with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
|
||||
if not path:
|
||||
path = '*' if self._request.method == 'OPTIONS' else '/'
|
||||
|
||||
# Make sure pseudo-headers comes before all the other headers
|
||||
headers = [
|
||||
(':method', self._request.method),
|
||||
(':authority', url.netloc),
|
||||
]
|
||||
|
||||
# The ":scheme" and ":path" pseudo-header fields MUST
|
||||
# be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
|
||||
if self._request.method != 'CONNECT':
|
||||
headers += [
|
||||
(':scheme', self._protocol.metadata['uri'].scheme),
|
||||
(':path', path),
|
||||
]
|
||||
|
||||
content_length = str(len(self._request.body))
|
||||
headers.append(('Content-Length', content_length))
|
||||
|
||||
content_length_name = self._request.headers.normkey(b'Content-Length')
|
||||
for name, values in self._request.headers.items():
|
||||
for value in values:
|
||||
value = str(value, 'utf-8')
|
||||
if name == content_length_name:
|
||||
if value != content_length:
|
||||
logger.warning(
|
||||
'Ignoring bad Content-Length header %r of request %r, '
|
||||
'sending %r instead',
|
||||
value,
|
||||
self._request,
|
||||
content_length,
|
||||
)
|
||||
continue
|
||||
headers.append((str(name, 'utf-8'), value))
|
||||
|
||||
return headers
|
||||
|
||||
def initiate_request(self) -> None:
|
||||
if self.check_request_url():
|
||||
headers = self._get_request_headers()
|
||||
self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
|
||||
self.metadata['request_sent'] = True
|
||||
self.send_data()
|
||||
else:
|
||||
# Close this stream calling the response errback
|
||||
# Note that we have not sent any headers
|
||||
self.close(StreamCloseReason.INVALID_HOSTNAME)
|
||||
|
||||
def send_data(self) -> None:
|
||||
"""Called immediately after the headers are sent. Here we send all the
|
||||
data as part of the request.
|
||||
|
||||
If the content length is 0 initially then we end the stream immediately and
|
||||
wait for response data.
|
||||
|
||||
Warning: Only call this method when stream not closed from client side
|
||||
and has initiated request already by sending HEADER frame. If not then
|
||||
stream will raise ProtocolError (raise by h2 state machine).
|
||||
"""
|
||||
if self.metadata['stream_closed_local']:
|
||||
raise StreamClosedError(self.stream_id)
|
||||
|
||||
# Firstly, check what the flow control window is for current stream.
|
||||
window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
|
||||
|
||||
# Next, check what the maximum frame size is.
|
||||
max_frame_size = self._protocol.conn.max_outbound_frame_size
|
||||
|
||||
# We will send no more than the window size or the remaining file size
|
||||
# of data in this call, whichever is smaller.
|
||||
bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
|
||||
|
||||
# We now need to send a number of data frames.
|
||||
while bytes_to_send_size > 0:
|
||||
chunk_size = min(bytes_to_send_size, max_frame_size)
|
||||
|
||||
data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
|
||||
data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
|
||||
|
||||
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
|
||||
|
||||
bytes_to_send_size -= chunk_size
|
||||
self.metadata['remaining_content_length'] -= chunk_size
|
||||
|
||||
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
|
||||
|
||||
# End the stream if no more data needs to be send
|
||||
if self.metadata['remaining_content_length'] == 0:
|
||||
self._protocol.conn.end_stream(self.stream_id)
|
||||
|
||||
# Q. What about the rest of the data?
|
||||
# Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
|
||||
|
||||
def receive_window_update(self) -> None:
|
||||
"""Flow control window size was changed.
|
||||
Send data that earlier could not be sent as we were
|
||||
blocked behind the flow control.
|
||||
"""
|
||||
if (
|
||||
self.metadata['remaining_content_length']
|
||||
and not self.metadata['stream_closed_server']
|
||||
and self.metadata['request_sent']
|
||||
):
|
||||
self.send_data()
|
||||
|
||||
def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
|
||||
self._response['body'].write(data)
|
||||
self._response['flow_controlled_size'] += flow_controlled_length
|
||||
|
||||
# We check maxsize here in case the Content-Length header was not received
|
||||
if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
|
||||
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
|
||||
return
|
||||
|
||||
if self._log_warnsize:
|
||||
self.metadata['reached_warnsize'] = True
|
||||
warning_msg = (
|
||||
f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
|
||||
f'warn size ({self._download_warnsize}) in request {self._request}'
|
||||
)
|
||||
logger.warning(warning_msg)
|
||||
|
||||
# Acknowledge the data received
|
||||
self._protocol.conn.acknowledge_received_data(
|
||||
self._response['flow_controlled_size'],
|
||||
self.stream_id
|
||||
)
|
||||
|
||||
def receive_headers(self, headers: List[HeaderTuple]) -> None:
|
||||
for name, value in headers:
|
||||
self._response['headers'][name] = value
|
||||
|
||||
# Check if we exceed the allowed max data size which can be received
|
||||
expected_size = int(self._response['headers'].get(b'Content-Length', -1))
|
||||
if self._download_maxsize and expected_size > self._download_maxsize:
|
||||
self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
|
||||
return
|
||||
|
||||
if self._log_warnsize:
|
||||
self.metadata['reached_warnsize'] = True
|
||||
warning_msg = (
|
||||
f'Expected response size ({expected_size}) larger than '
|
||||
f'download warn size ({self._download_warnsize}) in request {self._request}'
|
||||
)
|
||||
logger.warning(warning_msg)
|
||||
|
||||
def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
|
||||
"""Close this stream by sending a RST_FRAME to the remote peer"""
|
||||
if self.metadata['stream_closed_local']:
|
||||
raise StreamClosedError(self.stream_id)
|
||||
|
||||
# Clear buffer earlier to avoid keeping data in memory for a long time
|
||||
self._response['body'].truncate(0)
|
||||
|
||||
self.metadata['stream_closed_local'] = True
|
||||
self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
|
||||
self.close(reason)
|
||||
|
||||
def close(
|
||||
self,
|
||||
reason: StreamCloseReason,
|
||||
errors: Optional[List[BaseException]] = None,
|
||||
from_protocol: bool = False,
|
||||
) -> None:
|
||||
"""Based on the reason sent we will handle each case.
|
||||
"""
|
||||
if self.metadata['stream_closed_server']:
|
||||
raise StreamClosedError(self.stream_id)
|
||||
|
||||
if not isinstance(reason, StreamCloseReason):
|
||||
raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
|
||||
|
||||
# Have default value of errors as an empty list as
|
||||
# some cases can add a list of exceptions
|
||||
errors = errors or []
|
||||
|
||||
if not from_protocol:
|
||||
self._protocol.pop_stream(self.stream_id)
|
||||
|
||||
self.metadata['stream_closed_server'] = True
|
||||
|
||||
# We do not check for Content-Length or Transfer-Encoding in response headers
|
||||
# and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
|
||||
# a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6)
|
||||
|
||||
# NOTE: Order of handling the events is important here
|
||||
# As we immediately cancel the request when maxsize is exceeded while
|
||||
# receiving DATA_FRAME's when we have received the headers (not
|
||||
# having Content-Length)
|
||||
if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
|
||||
expected_size = int(self._response['headers'].get(
|
||||
b'Content-Length',
|
||||
self._response['flow_controlled_size'])
|
||||
)
|
||||
error_msg = (
|
||||
f'Cancelling download of {self._request.url}: received response '
|
||||
f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
|
||||
)
|
||||
logger.error(error_msg)
|
||||
self._deferred_response.errback(CancelledError(error_msg))
|
||||
|
||||
elif reason is StreamCloseReason.ENDED:
|
||||
self._fire_response_deferred()
|
||||
|
||||
# Stream was abruptly ended here
|
||||
elif reason is StreamCloseReason.CANCELLED:
|
||||
# Client has cancelled the request. Remove all the data
|
||||
# received and fire the response deferred with no flags set
|
||||
|
||||
# NOTE: The data is already flushed in Stream.reset_stream() called
|
||||
# immediately when the stream needs to be cancelled
|
||||
|
||||
# There maybe no :status in headers, we make
|
||||
# HTTP Status Code: 499 - Client Closed Request
|
||||
self._response['headers'][':status'] = '499'
|
||||
self._fire_response_deferred()
|
||||
|
||||
elif reason is StreamCloseReason.RESET:
|
||||
self._deferred_response.errback(ResponseFailed([
|
||||
Failure(
|
||||
f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
|
||||
ProtocolError
|
||||
)
|
||||
]))
|
||||
|
||||
elif reason is StreamCloseReason.CONNECTION_LOST:
|
||||
self._deferred_response.errback(ResponseFailed(errors))
|
||||
|
||||
elif reason is StreamCloseReason.INACTIVE:
|
||||
errors.insert(0, InactiveStreamClosed(self._request))
|
||||
self._deferred_response.errback(ResponseFailed(errors))
|
||||
|
||||
else:
|
||||
assert reason is StreamCloseReason.INVALID_HOSTNAME
|
||||
self._deferred_response.errback(InvalidHostname(
|
||||
self._request,
|
||||
str(self._protocol.metadata['uri'].host, 'utf-8'),
|
||||
f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
|
||||
))
|
||||
|
||||
def _fire_response_deferred(self) -> None:
|
||||
"""Builds response from the self._response dict
|
||||
and fires the response deferred callback with the
|
||||
generated response instance"""
|
||||
|
||||
body = self._response['body'].getvalue()
|
||||
response_cls = responsetypes.from_args(
|
||||
headers=self._response['headers'],
|
||||
url=self._request.url,
|
||||
body=body,
|
||||
)
|
||||
|
||||
response = response_cls(
|
||||
url=self._request.url,
|
||||
status=int(self._response['headers'][':status']),
|
||||
headers=self._response['headers'],
|
||||
body=body,
|
||||
request=self._request,
|
||||
certificate=self._protocol.metadata['certificate'],
|
||||
ip_address=self._protocol.metadata['ip_address'],
|
||||
protocol='h2',
|
||||
)
|
||||
|
||||
self._deferred_response.callback(response)
|
||||
|
|
@ -1,46 +1,179 @@
|
|||
import os
|
||||
import json
|
||||
import logging
|
||||
import warnings
|
||||
from os.path import join, exists
|
||||
import os
|
||||
from abc import abstractmethod
|
||||
from os.path import exists, join
|
||||
from typing import Optional, Type, TypeVar
|
||||
|
||||
from queuelib import PriorityQueue
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.utils.misc import load_object, create_instance
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Scheduler:
|
||||
class BaseSchedulerMeta(type):
|
||||
"""
|
||||
Scrapy Scheduler. It allows to enqueue requests and then get
|
||||
a next request to download. Scheduler is also handling duplication
|
||||
filtering, via dupefilter.
|
||||
|
||||
Prioritization and queueing is not performed by the Scheduler.
|
||||
User sets ``priority`` field for each Request, and a PriorityQueue
|
||||
(defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
|
||||
to dequeue requests in a desired order.
|
||||
|
||||
Scheduler uses two PriorityQueue instances, configured to work in-memory
|
||||
and on-disk (optional). When on-disk queue is present, it is used by
|
||||
default, and an in-memory queue is used as a fallback for cases where
|
||||
a disk queue can't handle a request (can't serialize it).
|
||||
|
||||
:setting:`SCHEDULER_MEMORY_QUEUE` and
|
||||
:setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
|
||||
which PriorityQueue instances would be instantiated with, to keep requests
|
||||
on disk and in memory respectively.
|
||||
|
||||
Overall, Scheduler is an object which holds several PriorityQueue instances
|
||||
(in-memory and on-disk) and implements fallback logic for them.
|
||||
Also, it handles dupefilters.
|
||||
Metaclass to check scheduler classes against the necessary interface
|
||||
"""
|
||||
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
|
||||
logunser=False, stats=None, pqclass=None, crawler=None):
|
||||
def __instancecheck__(cls, instance):
|
||||
return cls.__subclasscheck__(type(instance))
|
||||
|
||||
def __subclasscheck__(cls, subclass):
|
||||
return (
|
||||
hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
|
||||
and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
|
||||
and hasattr(subclass, "next_request") and callable(subclass.next_request)
|
||||
)
|
||||
|
||||
|
||||
class BaseScheduler(metaclass=BaseSchedulerMeta):
|
||||
"""
|
||||
The scheduler component is responsible for storing requests received from
|
||||
the engine, and feeding them back upon request (also to the engine).
|
||||
|
||||
The original sources of said requests are:
|
||||
|
||||
* Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks
|
||||
* Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods
|
||||
* Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods
|
||||
|
||||
The order in which the scheduler returns its stored requests (via the ``next_request`` method)
|
||||
plays a great part in determining the order in which those requests are downloaded.
|
||||
|
||||
The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler):
|
||||
"""
|
||||
Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
|
||||
"""
|
||||
return cls()
|
||||
|
||||
def open(self, spider: Spider) -> Optional[Deferred]:
|
||||
"""
|
||||
Called when the spider is opened by the engine. It receives the spider
|
||||
instance as argument and it's useful to execute initialization code.
|
||||
|
||||
:param spider: the spider object for the current crawl
|
||||
:type spider: :class:`~scrapy.spiders.Spider`
|
||||
"""
|
||||
pass
|
||||
|
||||
def close(self, reason: str) -> Optional[Deferred]:
|
||||
"""
|
||||
Called when the spider is closed by the engine. It receives the reason why the crawl
|
||||
finished as argument and it's useful to execute cleaning code.
|
||||
|
||||
:param reason: a string which describes the reason why the spider was closed
|
||||
:type reason: :class:`str`
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def has_pending_requests(self) -> bool:
|
||||
"""
|
||||
``True`` if the scheduler has enqueued requests, ``False`` otherwise
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def enqueue_request(self, request: Request) -> bool:
|
||||
"""
|
||||
Process a request received by the engine.
|
||||
|
||||
Return ``True`` if the request is stored correctly, ``False`` otherwise.
|
||||
|
||||
If ``False``, the engine will fire a ``request_dropped`` signal, and
|
||||
will not make further attempts to schedule the request at a later time.
|
||||
For reference, the default Scrapy scheduler returns ``False`` when the
|
||||
request is rejected by the dupefilter.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def next_request(self) -> Optional[Request]:
|
||||
"""
|
||||
Return the next :class:`~scrapy.http.Request` to be processed, or ``None``
|
||||
to indicate that there are no requests to be considered ready at the moment.
|
||||
|
||||
Returning ``None`` implies that no request from the scheduler will be sent
|
||||
to the downloader in the current reactor cycle. The engine will continue
|
||||
calling ``next_request`` until ``has_pending_requests`` is ``False``.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
|
||||
|
||||
|
||||
class Scheduler(BaseScheduler):
|
||||
"""
|
||||
Default Scrapy scheduler. This implementation also handles duplication
|
||||
filtering via the :setting:`dupefilter <DUPEFILTER_CLASS>`.
|
||||
|
||||
This scheduler stores requests into several priority queues (defined by the
|
||||
:setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues
|
||||
are backed by either memory or disk based queues (respectively defined by the
|
||||
:setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings).
|
||||
|
||||
Request prioritization is almost entirely delegated to the priority queue. The only
|
||||
prioritization performed by this scheduler is using the disk-based queue if present
|
||||
(i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based
|
||||
queue if a serialization error occurs. If the disk queue is not present, the memory one
|
||||
is used directly.
|
||||
|
||||
:param dupefilter: An object responsible for checking and filtering duplicate requests.
|
||||
The value for the :setting:`DUPEFILTER_CLASS` setting is used by default.
|
||||
:type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar:
|
||||
any class that implements the `BaseDupeFilter` interface
|
||||
|
||||
:param jobdir: The path of a directory to be used for persisting the crawl's state.
|
||||
The value for the :setting:`JOBDIR` setting is used by default.
|
||||
See :ref:`topics-jobs`.
|
||||
:type jobdir: :class:`str` or ``None``
|
||||
|
||||
:param dqclass: A class to be used as persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
|
||||
:type dqclass: class
|
||||
|
||||
:param mqclass: A class to be used as non-persistent request queue.
|
||||
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
|
||||
:type mqclass: class
|
||||
|
||||
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
|
||||
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
|
||||
:type logunser: bool
|
||||
|
||||
:param stats: A stats collector object to record stats about the request scheduling process.
|
||||
The value for the :setting:`STATS_CLASS` setting is used by default.
|
||||
:type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar:
|
||||
any class that implements the `StatsCollector` interface
|
||||
|
||||
:param pqclass: A class to be used as priority queue for requests.
|
||||
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
|
||||
:type pqclass: class
|
||||
|
||||
:param crawler: The crawler object corresponding to the current crawl.
|
||||
:type crawler: :class:`scrapy.crawler.Crawler`
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
dupefilter,
|
||||
jobdir: Optional[str] = None,
|
||||
dqclass=None,
|
||||
mqclass=None,
|
||||
logunser: bool = False,
|
||||
stats=None,
|
||||
pqclass=None,
|
||||
crawler: Optional[Crawler] = None,
|
||||
):
|
||||
self.df = dupefilter
|
||||
self.dqdir = self._dqdir(jobdir)
|
||||
self.pqclass = pqclass
|
||||
|
|
@ -51,42 +184,57 @@ class Scheduler:
|
|||
self.crawler = crawler
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
settings = crawler.settings
|
||||
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
|
||||
dupefilter = create_instance(dupefilter_cls, settings, crawler)
|
||||
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
|
||||
if pqclass is PriorityQueue:
|
||||
warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
|
||||
" is no longer supported because of API changes; "
|
||||
"please use 'scrapy.pqueues.ScrapyPriorityQueue'",
|
||||
ScrapyDeprecationWarning)
|
||||
from scrapy.pqueues import ScrapyPriorityQueue
|
||||
pqclass = ScrapyPriorityQueue
|
||||
def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
|
||||
"""
|
||||
Factory method, initializes the scheduler with arguments taken from the crawl settings
|
||||
"""
|
||||
dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
|
||||
return cls(
|
||||
dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
|
||||
jobdir=job_dir(crawler.settings),
|
||||
dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
|
||||
mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
|
||||
logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
|
||||
stats=crawler.stats,
|
||||
pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
|
||||
crawler=crawler,
|
||||
)
|
||||
|
||||
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
|
||||
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
|
||||
logunser = settings.getbool('SCHEDULER_DEBUG')
|
||||
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
|
||||
stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
|
||||
mqclass=mqclass, crawler=crawler)
|
||||
|
||||
def has_pending_requests(self):
|
||||
def has_pending_requests(self) -> bool:
|
||||
return len(self) > 0
|
||||
|
||||
def open(self, spider):
|
||||
def open(self, spider: Spider) -> Optional[Deferred]:
|
||||
"""
|
||||
(1) initialize the memory queue
|
||||
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
|
||||
(3) return the result of the dupefilter's ``open`` method
|
||||
"""
|
||||
self.spider = spider
|
||||
self.mqs = self._mq()
|
||||
self.dqs = self._dq() if self.dqdir else None
|
||||
return self.df.open()
|
||||
|
||||
def close(self, reason):
|
||||
if self.dqs:
|
||||
def close(self, reason: str) -> Optional[Deferred]:
|
||||
"""
|
||||
(1) dump pending requests to disk if there is a disk queue
|
||||
(2) return the result of the dupefilter's ``close`` method
|
||||
"""
|
||||
if self.dqs is not None:
|
||||
state = self.dqs.close()
|
||||
assert isinstance(self.dqdir, str)
|
||||
self._write_dqs_state(self.dqdir, state)
|
||||
return self.df.close(reason)
|
||||
|
||||
def enqueue_request(self, request):
|
||||
def enqueue_request(self, request: Request) -> bool:
|
||||
"""
|
||||
Unless the received request is filtered out by the Dupefilter, attempt to push
|
||||
it into the disk queue, falling back to pushing it into the memory queue.
|
||||
|
||||
Increment the appropriate stats, such as: ``scheduler/enqueued``,
|
||||
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
|
||||
|
||||
Return ``True`` if the request was stored successfully, ``False`` otherwise.
|
||||
"""
|
||||
if not request.dont_filter and self.df.request_seen(request):
|
||||
self.df.log(request, self.spider)
|
||||
return False
|
||||
|
|
@ -99,24 +247,35 @@ class Scheduler:
|
|||
self.stats.inc_value('scheduler/enqueued', spider=self.spider)
|
||||
return True
|
||||
|
||||
def next_request(self):
|
||||
def next_request(self) -> Optional[Request]:
|
||||
"""
|
||||
Return a :class:`~scrapy.http.Request` object from the memory queue,
|
||||
falling back to the disk queue if the memory queue is empty.
|
||||
Return ``None`` if there are no more enqueued requests.
|
||||
|
||||
Increment the appropriate stats, such as: ``scheduler/dequeued``,
|
||||
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
|
||||
"""
|
||||
request = self.mqs.pop()
|
||||
if request:
|
||||
if request is not None:
|
||||
self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
|
||||
else:
|
||||
request = self._dqpop()
|
||||
if request:
|
||||
if request is not None:
|
||||
self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
|
||||
if request:
|
||||
if request is not None:
|
||||
self.stats.inc_value('scheduler/dequeued', spider=self.spider)
|
||||
return request
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs)
|
||||
def __len__(self) -> int:
|
||||
"""
|
||||
Return the total amount of enqueued requests
|
||||
"""
|
||||
return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs)
|
||||
|
||||
def _dqpush(self, request):
|
||||
def _dqpush(self, request: Request) -> bool:
|
||||
if self.dqs is None:
|
||||
return
|
||||
return False
|
||||
try:
|
||||
self.dqs.push(request)
|
||||
except ValueError as e: # non serializable request
|
||||
|
|
@ -127,18 +286,18 @@ class Scheduler:
|
|||
logger.warning(msg, {'request': request, 'reason': e},
|
||||
exc_info=True, extra={'spider': self.spider})
|
||||
self.logunser = False
|
||||
self.stats.inc_value('scheduler/unserializable',
|
||||
spider=self.spider)
|
||||
return
|
||||
self.stats.inc_value('scheduler/unserializable', spider=self.spider)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def _mqpush(self, request):
|
||||
def _mqpush(self, request: Request) -> None:
|
||||
self.mqs.push(request)
|
||||
|
||||
def _dqpop(self):
|
||||
if self.dqs:
|
||||
def _dqpop(self) -> Optional[Request]:
|
||||
if self.dqs is not None:
|
||||
return self.dqs.pop()
|
||||
return None
|
||||
|
||||
def _mq(self):
|
||||
""" Create a new priority queue instance, with in-memory storage """
|
||||
|
|
@ -162,21 +321,22 @@ class Scheduler:
|
|||
{'queuesize': len(q)}, extra={'spider': self.spider})
|
||||
return q
|
||||
|
||||
def _dqdir(self, jobdir):
|
||||
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
|
||||
""" Return a folder name to keep disk queue state at """
|
||||
if jobdir:
|
||||
if jobdir is not None:
|
||||
dqdir = join(jobdir, 'requests.queue')
|
||||
if not exists(dqdir):
|
||||
os.makedirs(dqdir)
|
||||
return dqdir
|
||||
return None
|
||||
|
||||
def _read_dqs_state(self, dqdir):
|
||||
def _read_dqs_state(self, dqdir: str) -> list:
|
||||
path = join(dqdir, 'active.json')
|
||||
if not exists(path):
|
||||
return ()
|
||||
return []
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
def _write_dqs_state(self, dqdir, state):
|
||||
def _write_dqs_state(self, dqdir: str, state: list) -> None:
|
||||
with open(join(dqdir, 'active.json'), 'w') as f:
|
||||
json.dump(state, f)
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@ extracts information from them"""
|
|||
|
||||
import logging
|
||||
from collections import deque
|
||||
from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
from itemadapter import is_item
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -18,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
|
|||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -26,46 +30,46 @@ class Slot:
|
|||
|
||||
MIN_RESPONSE_SIZE = 1024
|
||||
|
||||
def __init__(self, max_active_size=5000000):
|
||||
def __init__(self, max_active_size: int = 5000000):
|
||||
self.max_active_size = max_active_size
|
||||
self.queue = deque()
|
||||
self.active = set()
|
||||
self.active_size = 0
|
||||
self.itemproc_size = 0
|
||||
self.closing = None
|
||||
self.queue: Deque[QueueTuple] = deque()
|
||||
self.active: Set[Request] = set()
|
||||
self.active_size: int = 0
|
||||
self.itemproc_size: int = 0
|
||||
self.closing: Optional[Deferred] = None
|
||||
|
||||
def add_response_request(self, response, request):
|
||||
deferred = defer.Deferred()
|
||||
self.queue.append((response, request, deferred))
|
||||
if isinstance(response, Response):
|
||||
self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE)
|
||||
def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
|
||||
deferred = Deferred()
|
||||
self.queue.append((result, request, deferred))
|
||||
if isinstance(result, Response):
|
||||
self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
|
||||
else:
|
||||
self.active_size += self.MIN_RESPONSE_SIZE
|
||||
return deferred
|
||||
|
||||
def next_response_request_deferred(self):
|
||||
def next_response_request_deferred(self) -> QueueTuple:
|
||||
response, request, deferred = self.queue.popleft()
|
||||
self.active.add(request)
|
||||
return response, request, deferred
|
||||
|
||||
def finish_response(self, response, request):
|
||||
def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
|
||||
self.active.remove(request)
|
||||
if isinstance(response, Response):
|
||||
self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE)
|
||||
if isinstance(result, Response):
|
||||
self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
|
||||
else:
|
||||
self.active_size -= self.MIN_RESPONSE_SIZE
|
||||
|
||||
def is_idle(self):
|
||||
def is_idle(self) -> bool:
|
||||
return not (self.queue or self.active)
|
||||
|
||||
def needs_backout(self):
|
||||
def needs_backout(self) -> bool:
|
||||
return self.active_size > self.max_active_size
|
||||
|
||||
|
||||
class Scraper:
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.slot = None
|
||||
self.slot: Optional[Slot] = None
|
||||
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
|
||||
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
|
||||
self.itemproc = itemproc_cls.from_crawler(crawler)
|
||||
|
|
@ -74,36 +78,39 @@ class Scraper:
|
|||
self.signals = crawler.signals
|
||||
self.logformatter = crawler.logformatter
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def open_spider(self, spider):
|
||||
@inlineCallbacks
|
||||
def open_spider(self, spider: Spider):
|
||||
"""Open the given spider for scraping and allocate resources for it"""
|
||||
self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
|
||||
yield self.itemproc.open_spider(spider)
|
||||
|
||||
def close_spider(self, spider):
|
||||
def close_spider(self, spider: Spider) -> Deferred:
|
||||
"""Close a spider being scraped and release its resources"""
|
||||
slot = self.slot
|
||||
slot.closing = defer.Deferred()
|
||||
slot.closing.addCallback(self.itemproc.close_spider)
|
||||
self._check_if_closing(spider, slot)
|
||||
return slot.closing
|
||||
if self.slot is None:
|
||||
raise RuntimeError("Scraper slot not assigned")
|
||||
self.slot.closing = Deferred()
|
||||
self.slot.closing.addCallback(self.itemproc.close_spider)
|
||||
self._check_if_closing(spider)
|
||||
return self.slot.closing
|
||||
|
||||
def is_idle(self):
|
||||
def is_idle(self) -> bool:
|
||||
"""Return True if there isn't any more spiders to process"""
|
||||
return not self.slot
|
||||
|
||||
def _check_if_closing(self, spider, slot):
|
||||
if slot.closing and slot.is_idle():
|
||||
slot.closing.callback(spider)
|
||||
def _check_if_closing(self, spider: Spider) -> None:
|
||||
assert self.slot is not None # typing
|
||||
if self.slot.closing and self.slot.is_idle():
|
||||
self.slot.closing.callback(spider)
|
||||
|
||||
def enqueue_scrape(self, response, request, spider):
|
||||
slot = self.slot
|
||||
dfd = slot.add_response_request(response, request)
|
||||
def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
|
||||
if self.slot is None:
|
||||
raise RuntimeError("Scraper slot not assigned")
|
||||
dfd = self.slot.add_response_request(result, request)
|
||||
|
||||
def finish_scraping(_):
|
||||
slot.finish_response(response, request)
|
||||
self._check_if_closing(spider, slot)
|
||||
self._scrape_next(spider, slot)
|
||||
self.slot.finish_response(result, request)
|
||||
self._check_if_closing(spider)
|
||||
self._scrape_next(spider)
|
||||
return _
|
||||
|
||||
dfd.addBoth(finish_scraping)
|
||||
|
|
@ -112,15 +119,16 @@ class Scraper:
|
|||
{'request': request},
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
self._scrape_next(spider, slot)
|
||||
self._scrape_next(spider)
|
||||
return dfd
|
||||
|
||||
def _scrape_next(self, spider, slot):
|
||||
while slot.queue:
|
||||
response, request, deferred = slot.next_response_request_deferred()
|
||||
def _scrape_next(self, spider: Spider) -> None:
|
||||
assert self.slot is not None # typing
|
||||
while self.slot.queue:
|
||||
response, request, deferred = self.slot.next_response_request_deferred()
|
||||
self._scrape(response, request, spider).chainDeferred(deferred)
|
||||
|
||||
def _scrape(self, result, request, spider):
|
||||
def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
|
||||
"""
|
||||
Handle the downloaded response or failure through the spider callback/errback
|
||||
"""
|
||||
|
|
@ -131,7 +139,7 @@ class Scraper:
|
|||
dfd.addCallback(self.handle_spider_output, request, result, spider)
|
||||
return dfd
|
||||
|
||||
def _scrape2(self, result, request, spider):
|
||||
def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
|
||||
"""
|
||||
Handle the different cases of request's result been a Response or a Failure
|
||||
"""
|
||||
|
|
@ -141,14 +149,14 @@ class Scraper:
|
|||
dfd = self.call_spider(result, request, spider)
|
||||
return dfd.addErrback(self._log_download_errors, result, request, spider)
|
||||
|
||||
def call_spider(self, result, request, spider):
|
||||
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
|
||||
if isinstance(result, Response):
|
||||
if getattr(result, "request", None) is None:
|
||||
result.request = request
|
||||
callback = result.request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
dfd = defer_succeed(result)
|
||||
dfd.addCallback(callback, **result.request.cb_kwargs)
|
||||
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
|
||||
else: # result is a Failure
|
||||
result.request = request
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
|
|
@ -156,7 +164,7 @@ class Scraper:
|
|||
dfd.addErrback(request.errback)
|
||||
return dfd.addCallback(iterate_spider_output)
|
||||
|
||||
def handle_spider_error(self, _failure, request, response, spider):
|
||||
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
|
||||
exc = _failure.value
|
||||
if isinstance(exc, CloseSpider):
|
||||
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
|
||||
|
|
@ -177,7 +185,7 @@ class Scraper:
|
|||
spider=spider
|
||||
)
|
||||
|
||||
def handle_spider_output(self, result, request, response, spider):
|
||||
def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
|
||||
if not result:
|
||||
return defer_succeed(None)
|
||||
it = iter_errback(result, self.handle_spider_error, request, response, spider)
|
||||
|
|
@ -185,12 +193,14 @@ class Scraper:
|
|||
request, response, spider)
|
||||
return dfd
|
||||
|
||||
def _process_spidermw_output(self, output, request, response, spider):
|
||||
def _process_spidermw_output(self, output: Any, request: Request, response: Response,
|
||||
spider: Spider) -> Optional[Deferred]:
|
||||
"""Process each Request/Item (given in the output parameter) returned
|
||||
from the given spider
|
||||
"""
|
||||
assert self.slot is not None # typing
|
||||
if isinstance(output, Request):
|
||||
self.crawler.engine.crawl(request=output, spider=spider)
|
||||
self.crawler.engine.crawl(request=output)
|
||||
elif is_item(output):
|
||||
self.slot.itemproc_size += 1
|
||||
dfd = self.itemproc.process_item(output, spider)
|
||||
|
|
@ -205,12 +215,18 @@ class Scraper:
|
|||
{'request': request, 'typename': typename},
|
||||
extra={'spider': spider},
|
||||
)
|
||||
return None
|
||||
|
||||
def _log_download_errors(self, spider_failure, download_failure, request, spider):
|
||||
def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,
|
||||
spider: Spider) -> Union[Failure, None]:
|
||||
"""Log and silence errors that come from the engine (typically download
|
||||
errors that got propagated thru here)
|
||||
errors that got propagated thru here).
|
||||
|
||||
spider_failure: the value passed into the errback of self.call_spider()
|
||||
download_failure: the value passed into _scrape2() from
|
||||
ExecutionEngine._handle_downloader_output() as "result"
|
||||
"""
|
||||
if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
|
||||
if not download_failure.check(IgnoreRequest):
|
||||
if download_failure.frames:
|
||||
logkws = self.logformatter.download_error(download_failure, request, spider)
|
||||
logger.log(
|
||||
|
|
@ -230,10 +246,12 @@ class Scraper:
|
|||
|
||||
if spider_failure is not download_failure:
|
||||
return spider_failure
|
||||
return None
|
||||
|
||||
def _itemproc_finished(self, output, item, response, spider):
|
||||
def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:
|
||||
"""ItemProcessor finished for the given ``item`` and returned ``output``
|
||||
"""
|
||||
assert self.slot is not None # typing
|
||||
self.slot.itemproc_size -= 1
|
||||
if isinstance(output, Failure):
|
||||
ex = output.value
|
||||
|
|
|
|||
|
|
@ -4,22 +4,25 @@ Spider Middleware manager
|
|||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
from itertools import islice
|
||||
from typing import Any, Callable, Generator, Iterable, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
from scrapy.http import Response
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
from scrapy.utils.python import MutableChain
|
||||
|
||||
|
||||
def _isiterable(possible_iterator):
|
||||
return hasattr(possible_iterator, '__iter__')
|
||||
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
|
||||
|
||||
|
||||
def _fname(f):
|
||||
return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
|
||||
def _isiterable(o) -> bool:
|
||||
return isinstance(o, Iterable)
|
||||
|
||||
|
||||
class SpiderMiddlewareManager(MiddlewareManager):
|
||||
|
|
@ -41,88 +44,100 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
process_spider_exception = getattr(mw, 'process_spider_exception', None)
|
||||
self.methods['process_spider_exception'].appendleft(process_spider_exception)
|
||||
|
||||
def scrape_response(self, scrape_func, response, request, spider):
|
||||
|
||||
def process_spider_input(response):
|
||||
for method in self.methods['process_spider_input']:
|
||||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
msg = (f"Middleware {_fname(method)} must return None "
|
||||
f"or raise an exception, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
except _InvalidOutput:
|
||||
raise
|
||||
except Exception:
|
||||
return scrape_func(Failure(), request, spider)
|
||||
return scrape_func(response, request, spider)
|
||||
|
||||
def _evaluate_iterable(iterable, exception_processor_index, recover_to):
|
||||
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
|
||||
spider: Spider) -> Any:
|
||||
for method in self.methods['process_spider_input']:
|
||||
method = cast(Callable, method)
|
||||
try:
|
||||
for r in iterable:
|
||||
yield r
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
msg = (f"Middleware {method.__qualname__} must return None "
|
||||
f"or raise an exception, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
except _InvalidOutput:
|
||||
raise
|
||||
except Exception:
|
||||
return scrape_func(Failure(), request, spider)
|
||||
return scrape_func(response, request, spider)
|
||||
|
||||
def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
|
||||
exception_processor_index: int, recover_to: MutableChain) -> Generator:
|
||||
try:
|
||||
for r in iterable:
|
||||
yield r
|
||||
except Exception as ex:
|
||||
exception_result = self._process_spider_exception(response, spider, Failure(ex),
|
||||
exception_processor_index)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
recover_to.extend(exception_result)
|
||||
|
||||
def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
|
||||
start_index: int = 0) -> Union[Failure, MutableChain]:
|
||||
exception = _failure.value
|
||||
# don't handle _InvalidOutput exception
|
||||
if isinstance(exception, _InvalidOutput):
|
||||
return _failure
|
||||
method_list = islice(self.methods['process_spider_exception'], start_index, None)
|
||||
for method_index, method in enumerate(method_list, start=start_index):
|
||||
if method is None:
|
||||
continue
|
||||
result = method(response=response, exception=exception, spider=spider)
|
||||
if _isiterable(result):
|
||||
# stop exception handling by handing control over to the
|
||||
# process_spider_output chain if an iterable has been returned
|
||||
return self._process_spider_output(response, spider, result, method_index + 1)
|
||||
elif result is None:
|
||||
continue
|
||||
else:
|
||||
msg = (f"Middleware {method.__qualname__} must return None "
|
||||
f"or an iterable, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
return _failure
|
||||
|
||||
def _process_spider_output(self, response: Response, spider: Spider,
|
||||
result: Iterable, start_index: int = 0) -> MutableChain:
|
||||
# items in this iterable do not need to go through the process_spider_output
|
||||
# chain, they went through it already from the process_spider_exception method
|
||||
recovered = MutableChain()
|
||||
|
||||
method_list = islice(self.methods['process_spider_output'], start_index, None)
|
||||
for method_index, method in enumerate(method_list, start=start_index):
|
||||
if method is None:
|
||||
continue
|
||||
try:
|
||||
# might fail directly if the output value is not a generator
|
||||
result = method(response=response, result=result, spider=spider)
|
||||
except Exception as ex:
|
||||
exception_result = process_spider_exception(Failure(ex), exception_processor_index)
|
||||
exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
recover_to.extend(exception_result)
|
||||
return exception_result
|
||||
if _isiterable(result):
|
||||
result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
|
||||
else:
|
||||
msg = (f"Middleware {method.__qualname__} must return an "
|
||||
f"iterable, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
|
||||
def process_spider_exception(_failure, start_index=0):
|
||||
exception = _failure.value
|
||||
# don't handle _InvalidOutput exception
|
||||
if isinstance(exception, _InvalidOutput):
|
||||
return _failure
|
||||
method_list = islice(self.methods['process_spider_exception'], start_index, None)
|
||||
for method_index, method in enumerate(method_list, start=start_index):
|
||||
if method is None:
|
||||
continue
|
||||
result = method(response=response, exception=exception, spider=spider)
|
||||
if _isiterable(result):
|
||||
# stop exception handling by handing control over to the
|
||||
# process_spider_output chain if an iterable has been returned
|
||||
return process_spider_output(result, method_index + 1)
|
||||
elif result is None:
|
||||
continue
|
||||
else:
|
||||
msg = (f"Middleware {_fname(method)} must return None "
|
||||
f"or an iterable, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
return _failure
|
||||
return MutableChain(result, recovered)
|
||||
|
||||
def process_spider_output(result, start_index=0):
|
||||
# items in this iterable do not need to go through the process_spider_output
|
||||
# chain, they went through it already from the process_spider_exception method
|
||||
recovered = MutableChain()
|
||||
def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
|
||||
recovered = MutableChain()
|
||||
result = self._evaluate_iterable(response, spider, result, 0, recovered)
|
||||
return MutableChain(self._process_spider_output(response, spider, result), recovered)
|
||||
|
||||
method_list = islice(self.methods['process_spider_output'], start_index, None)
|
||||
for method_index, method in enumerate(method_list, start=start_index):
|
||||
if method is None:
|
||||
continue
|
||||
try:
|
||||
# might fail directly if the output value is not a generator
|
||||
result = method(response=response, result=result, spider=spider)
|
||||
except Exception as ex:
|
||||
exception_result = process_spider_exception(Failure(ex), method_index + 1)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
return exception_result
|
||||
if _isiterable(result):
|
||||
result = _evaluate_iterable(result, method_index + 1, recovered)
|
||||
else:
|
||||
msg = (f"Middleware {_fname(method)} must return an "
|
||||
f"iterable, got {type(result)}")
|
||||
raise _InvalidOutput(msg)
|
||||
def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
|
||||
spider: Spider) -> Deferred:
|
||||
def process_callback_output(result: Iterable) -> MutableChain:
|
||||
return self._process_callback_output(response, spider, result)
|
||||
|
||||
return MutableChain(result, recovered)
|
||||
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
|
||||
return self._process_spider_exception(response, spider, _failure)
|
||||
|
||||
def process_callback_output(result):
|
||||
recovered = MutableChain()
|
||||
result = _evaluate_iterable(result, 0, recovered)
|
||||
return MutableChain(process_spider_output(result), recovered)
|
||||
|
||||
dfd = mustbe_deferred(process_spider_input, response)
|
||||
dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
|
||||
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
|
||||
return dfd
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
|
||||
return self._process_chain('process_start_requests', start_requests, spider)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.utils.log import (
|
|||
configure_logging,
|
||||
get_scrapy_root_handler,
|
||||
install_scrapy_root_handler,
|
||||
log_reactor_info,
|
||||
log_scrapy_info,
|
||||
LogCounterHandler,
|
||||
)
|
||||
|
|
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class Crawler:
|
||||
|
||||
def __init__(self, spidercls, settings=None):
|
||||
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
|
||||
if isinstance(spidercls, Spider):
|
||||
raise ValueError('The spidercls argument must be a class, not an object')
|
||||
|
||||
|
|
@ -50,6 +51,7 @@ class Crawler:
|
|||
self.spidercls.update_settings(self.settings)
|
||||
|
||||
self.signals = SignalManager(self)
|
||||
|
||||
self.stats = load_object(self.settings['STATS_CLASS'])(self)
|
||||
|
||||
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
|
||||
|
|
@ -69,6 +71,26 @@ class Crawler:
|
|||
|
||||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
self.request_fingerprinter = create_instance(
|
||||
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
|
||||
settings=self.settings,
|
||||
crawler=self,
|
||||
)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
# but before something imports twisted.internet.reactor
|
||||
if reactor_class:
|
||||
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
else:
|
||||
from twisted.internet import default
|
||||
default.install()
|
||||
log_reactor_info()
|
||||
if reactor_class:
|
||||
verify_installed_reactor(reactor_class)
|
||||
|
||||
self.extensions = ExtensionManager.from_crawler(self)
|
||||
|
||||
self.settings.freeze()
|
||||
|
|
@ -153,7 +175,6 @@ class CrawlerRunner:
|
|||
self._crawlers = set()
|
||||
self._active = set()
|
||||
self.bootstrap_failed = False
|
||||
self._handle_twisted_reactor()
|
||||
|
||||
@property
|
||||
def spiders(self):
|
||||
|
|
@ -247,10 +268,6 @@ class CrawlerRunner:
|
|||
while self._active:
|
||||
yield defer.DeferredList(self._active)
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
|
||||
|
||||
|
||||
class CrawlerProcess(CrawlerRunner):
|
||||
"""
|
||||
|
|
@ -278,7 +295,6 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
||||
|
|
@ -298,7 +314,12 @@ class CrawlerProcess(CrawlerRunner):
|
|||
{'signame': signame})
|
||||
reactor.callFromThread(self._stop_reactor)
|
||||
|
||||
def start(self, stop_after_crawl=True):
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings, init_reactor=True)
|
||||
|
||||
def start(self, stop_after_crawl=True, install_signal_handlers=True):
|
||||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
|
|
@ -309,6 +330,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
||||
:param bool install_signal_handlers: whether to install the shutdown
|
||||
handlers (default: True)
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
if stop_after_crawl:
|
||||
|
|
@ -318,6 +342,8 @@ class CrawlerProcess(CrawlerRunner):
|
|||
return
|
||||
d.addBoth(self._stop_reactor)
|
||||
|
||||
if install_signal_handlers:
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
resolver_class = load_object(self.settings["DNS_RESOLVER"])
|
||||
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
|
||||
resolver.install_on_reactor()
|
||||
|
|
@ -337,8 +363,3 @@ class CrawlerProcess(CrawlerRunner):
|
|||
reactor.stop()
|
||||
except RuntimeError: # raised if already stopped or in shutdown stage
|
||||
pass
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue