RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into integrate-mime

This commit is contained in:
Adrián Chaves 2023-01-02 13:14:39 +01:00
parent 42bca211d9 deaf1fb6cf
commit b336980bd1
216 changed files with 5044 additions and 2425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.6.1
current_version = 2.7.1
commit = True
tag = True
tag_name = {new_version}

19
.flake8
View File

@ -4,16 +4,19 @@ max-line-length = 119
ignore = W503
exclude =
docs/conf.py
per-file-ignores =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
scrapy/__init__.py:E402
scrapy/core/downloader/handlers/http.py:F401
scrapy/http/__init__.py:F401
scrapy/linkextractors/__init__.py:E402,F401
scrapy/selector/__init__.py:F401
scrapy/spiders/__init__.py:E402,F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
scrapy/utils/url.py:F403,F405
tests/test_loader.py:E741

15
.github/workflows/checks.yml vendored
View File

@ -3,15 +3,15 @@ on: [push, pull_request]
jobs:
checks:
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: security
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
@ -22,15 +22,18 @@ jobs:
- python-version: 3.7
env:
TOXENV: typing
- python-version: "3.10" # Keep in sync with .readthedocs.yml
- python-version: "3.11" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
- python-version: "3.11"
env:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

12
.github/workflows/publish.yml vendored
View File

@ -3,16 +3,16 @@ on: [push]
jobs:
publish:
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.11"
- name: Check Tag
id: check-release-tag
@ -24,8 +24,8 @@ jobs:
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
pip install --upgrade build twine
python -m build
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*

8
.github/workflows/tests-macos.yml vendored
View File

@ -3,17 +3,17 @@ on: [push, pull_request]
jobs:
tests:
runs-on: macos-10.15
runs-on: macos-11
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

29
.github/workflows/tests-ubuntu.yml vendored
View File

@ -3,7 +3,7 @@ on: [push, pull_request]
jobs:
tests:
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
@ -17,13 +17,15 @@ jobs:
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: asyncio
- python-version: pypy3
- python-version: pypy3.9
env:
TOXENV: pypy3
PYPY_VERSION: 3.9-v7.3.9
# pinned deps
- python-version: 3.7.13
@ -32,10 +34,9 @@ jobs:
- python-version: 3.7.13
env:
TOXENV: asyncio-pinned
- python-version: pypy3
- python-version: pypy3.7
env:
TOXENV: pypy3-pinned
PYPY_VERSION: 3.7-v7.3.5
# extras
# extra-deps includes reppy, which does not support Python 3.9
@ -45,30 +46,22 @@ jobs:
TOXENV: extra-deps
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
run: |
sudo apt-get update
# libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
sudo apt-get install libxml2-dev libxslt-dev
- name: Run tests
env: ${{ matrix.env }}
run: |
if [[ ! -z "$PYPY_VERSION" ]]; then
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
pip install -U tox
tox

11
.github/workflows/tests-windows.yml vendored
View File

@ -23,12 +23,19 @@ jobs:
- python-version: "3.10"
env:
TOXENV: asyncio
# no binary package for lxml for 3.11 yet
# - python-version: "3.11"
# env:
# TOXENV: py
# - python-version: "3.11"
# env:
# TOXENV: asyncio
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

3
.gitignore vendored
View File

@ -23,3 +23,6 @@ test-output.*
# Windows
Thumbs.db
# OSX miscellaneous
.DS_Store

2
.readthedocs.yml
View File

@ -9,7 +9,7 @@ build:
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.10" # Keep in sync with .github/workflows/checks.yml
python: "3.11" # Keep in sync with .github/workflows/checks.yml
python:
install:

152
CODE_OF_CONDUCT.md
View File

@ -1,77 +1,133 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
Examples of behavior that contributes to a positive environment for our
community include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior by participants include:
Examples of unacceptable behavior include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
## Enforcement Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at opensource@zyte.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
reported to the community leaders responsible for enforcement at
opensource@zyte.com.
All complaints will be reviewed and investigated promptly and fairly.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version].
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations

4
INSTALL
View File

@ -1,4 +0,0 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
* https://docs.scrapy.org/en/latest/intro/install.html (online version)

4
INSTALL.md Normal file
View File

@ -0,0 +1,4 @@
For information about installing Scrapy see:
* [Local docs](docs/intro/install.rst)
* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)

8
README.rst
View File

@ -1,4 +1,5 @@
.. image:: https://scrapy.org/img/scrapylogo.png
:target: https://scrapy.org/
======
Scrapy
@ -63,7 +64,9 @@ Requirements
Install
=======
The quick way::
The quick way:
.. code:: bash
pip install scrapy
@ -94,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
Please note that this project is released with a Contributor Code of Conduct
(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
By participating in this project you agree to abide by its terms.
Please report unacceptable behavior to opensource@zyte.com.

12
conftest.py
View File

@ -21,7 +21,7 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
with open('tests/ignores.txt') as reader:
with Path('tests/ignores.txt').open(encoding="utf-8") as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
@ -42,16 +42,6 @@ def chdir(tmpdir):
tmpdir.chdir()
def pytest_collection_modifyitems(session, config, items):
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
try:
from pytest_flake8 import Flake8Item
if config.getoption('--flake8'):
items[:] = [item for item in items if isinstance(item, Flake8Item)]
except ImportError:
pass
def pytest_addoption(parser):
parser.addoption(
"--reactor",

4
docs/Makefile
View File

@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
$(PYTHON) -c "import webbrowser; from pathlib import Path; \
webbrowser.open('file://' + Path('build/html/index.html').resolve())"
clean:
-rm -rf build/*

28
docs/_ext/scrapydocs.py
View File

@ -1,8 +1,8 @@
from operator import itemgetter
from docutils.parsers.rst.roles import set_classes
from docutils import nodes
from docutils.parsers.rst import Directive
from sphinx.util.nodes import make_refnode
from operator import itemgetter
class settingslist_node(nodes.General, nodes.Element):
@ -15,7 +15,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
if node.tagname == 'index' and node['entries']:
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
def setup(app):
app.add_crossref_type(
directivename = "setting",
rolename = "setting",
indextemplate = "pair: %s; setting",
directivename="setting",
rolename="setting",
indextemplate="pair: %s; setting",
)
app.add_crossref_type(
directivename = "signal",
rolename = "signal",
indextemplate = "pair: %s; signal",
directivename="signal",
rolename="signal",
indextemplate="pair: %s; signal",
)
app.add_crossref_type(
directivename = "command",
rolename = "command",
indextemplate = "pair: %s; command",
directivename="command",
rolename="command",
indextemplate="pair: %s; command",
)
app.add_crossref_type(
directivename = "reqmeta",
rolename = "reqmeta",
indextemplate = "pair: %s; reqmeta",
directivename="reqmeta",
rolename="reqmeta",
indextemplate="pair: %s; reqmeta",
)
app.add_role('source', source_role)
app.add_role('commit', commit_role)

15
docs/conf.py
View File

@ -11,13 +11,12 @@
import sys
from datetime import datetime
from os import path
from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
sys.path.append(path.join(path.dirname(__file__), "_ext"))
sys.path.insert(0, path.dirname(path.dirname(__file__)))
# is relative to the documentation root, use Path.absolute to make it absolute.
sys.path.append(str(Path(__file__).parent / "_ext"))
sys.path.insert(0, str(Path(__file__).parent.parent))
# General configuration
@ -291,9 +290,9 @@ intersphinx_mapping = {
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
'tox': ('https://tox.wiki/en/latest/', None),
'twisted': ('https://docs.twisted.org/en/stable/', None),
'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
intersphinx_disabled_reftypes = []

17
docs/conftest.py
View File

@ -1,21 +1,22 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
from scrapy.http.response.html import HtmlResponse
def load_response(url, filename):
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
with open(input_path, 'rb') as input_file:
return HtmlResponse(url, body=input_file.read())
def load_response(url: str, filename: str) -> HtmlResponse:
input_path = Path(__file__).parent / '_tests' / filename
return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):

2
docs/contributing.rst
View File

@ -214,7 +214,7 @@ Tests
=====
Tests are implemented using the :doc:`Twisted unit-testing framework
<twisted:core/development/policy/test-standard>`. Running tests requires
<twisted:development/test-standard>`. Running tests requires
:doc:`tox <tox:index>`.
.. _running-tests:

17
docs/index.rst
View File

@ -130,7 +130,6 @@ Built-in services
topics/stats
topics/email
topics/telnetconsole
topics/webservice
:doc:`topics/logging`
Learn how to use Python's builtin logging on Scrapy.
@ -144,9 +143,6 @@ Built-in services
:doc:`topics/telnetconsole`
Inspect a running crawler using a built-in Python console.
:doc:`topics/webservice`
Monitor and control a crawler using a web service.
Solving specific problems
=========================
@ -229,10 +225,11 @@ Extending Scrapy
topics/downloader-middleware
topics/spider-middleware
topics/extensions
topics/api
topics/signals
topics/scheduler
topics/exporters
topics/components
topics/api
:doc:`topics/architecture`
@ -247,9 +244,6 @@ Extending Scrapy
:doc:`topics/extensions`
Extend Scrapy with your custom functionality
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality
:doc:`topics/signals`
See all available signals and how to work with them.
@ -259,6 +253,13 @@ Extending Scrapy
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).
:doc:`topics/components`
Learn the common API and some good practices when building custom Scrapy
components.
:doc:`topics/api`
Use it on extensions and middlewares to extend Scrapy functionality.
All the rest
============

8
docs/intro/install.rst
View File

@ -10,7 +10,7 @@ Supported Python versions
=========================
Scrapy requires Python 3.7+, either the CPython implementation (default) or
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
Some of these packages themselves depends on non-Python packages
Some of these packages themselves depend on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
@ -187,7 +187,7 @@ solutions:
* Install `homebrew`_ following the instructions in https://brew.sh/
* Update your ``PATH`` variable to state that homebrew packages should be
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
if you're using `zsh`_ as default shell)::
echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
PyPy
----
We recommend using the latest PyPy version. The version tested is 5.9.0.
We recommend using the latest PyPy version.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.

12
docs/intro/tutorial.rst
View File

@ -85,6 +85,8 @@ page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
from pathlib import Path
import scrapy
@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
from pathlib import Path
import scrapy
@ -191,8 +194,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy
@ -379,7 +381,7 @@ like this:
Let's open up scrapy shell and play a bit to find out how to extract the data
we want::
$ scrapy shell 'https://quotes.toscrape.com'
scrapy shell 'https://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with:

399
docs/news.rst
View File

@ -3,6 +3,345 @@
Release notes
=============
.. _release-2.7.1:
Scrapy 2.7.1 (2022-11-02)
-------------------------
New features
~~~~~~~~~~~~
- Relaxed the restriction introduced in 2.6.2 so that the
``Proxy-Authorization`` header can again be set explicitly, as long as the
proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
for as long as that proxy URL remains the same; this restores compatibility
with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`).
Bug fixes
~~~~~~~~~
- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options
together now produces an error instead of ignoring the former option
(:issue:`5516`, :issue:`5605`).
- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current
event loop with code that explicitly requests a loop from the event loop
policy (:issue:`5685`, :issue:`5689`).
- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`,
:issue:`5589`).
- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`).
- Improved code that checks if generators return values, so that it no longer
fails on decorated methods and partial methods (:issue:`5323`,
:issue:`5592`, :issue:`5599`, :issue:`5691`).
Documentation
~~~~~~~~~~~~~
- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`).
- Fixed typos (:issue:`5681`, :issue:`5694`).
Quality assurance
~~~~~~~~~~~~~~~~~
- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`).
- Ignored harmless deprecation warnings from :mod:`typing` in tests
(:issue:`5686`, :issue:`5697`).
- Modernized our CI configuration (:issue:`5695`, :issue:`5696`).
.. _release-2.7.0:
Scrapy 2.7.0 (2022-10-17)
-----------------------------
Highlights:
- Added Python 3.11 support, dropped Python 3.6 support
- Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
- :ref:`Asyncio support <using-asyncio>` is enabled by default on new
projects
- Output names of item fields can now be arbitrary strings
- Centralized :ref:`request fingerprinting <request-fingerprints>`
configuration is now possible
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
Support for the upcoming Python 3.11 has been added.
The minimum required version of some dependencies has changed as well:
- lxml_: 3.5.0 → 4.3.0
- Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
- zope.interface_: 5.0.0 → 5.1.0
(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
:issue:`5670`, :issue:`5678`)
Deprecations
~~~~~~~~~~~~
- :meth:`ImagesPipeline.thumb_path
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
``item`` parameter (:issue:`5504`, :issue:`5508`).
- The ``scrapy.downloadermiddlewares.decompression`` module is now
deprecated (:issue:`5546`, :issue:`5547`).
New features
~~~~~~~~~~~~
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>` can now be
defined as an :term:`asynchronous generator` (:issue:`4978`).
- The output of :class:`~scrapy.Request` callbacks defined as
:ref:`coroutines <topics-coroutines>` is now processed asynchronously
(:issue:`4978`).
- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
callbacks <topics-coroutines>` (:issue:`5657`).
- New projects created with the :command:`startproject` command have
:ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
:issue:`5679`).
- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
dictionary to customize the output name of item fields, lifting the
restriction that required output names to be valid Python identifiers, e.g.
preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
:issue:`3696`).
- You can now customize :ref:`request fingerprinting <request-fingerprints>`
through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
having to change it on every Scrapy component that relies on request
fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
:issue:`4524`).
- ``jsonl`` is now supported and encouraged as a file extension for `JSON
Lines`_ files (:issue:`4848`).
.. _JSON Lines: https://jsonlines.org/
- :meth:`ImagesPipeline.thumb_path
<scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
Bug fixes
~~~~~~~~~
- When using Google Cloud Storage with a :ref:`media pipeline
<topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
:setting:`FILES_STORE` does not point at the root of your Google Cloud
Storage bucket (:issue:`5317`, :issue:`5318`).
- The :command:`parse` command now supports :ref:`asynchronous callbacks
<topics-coroutines>` (:issue:`5424`, :issue:`5577`).
- When using the :command:`parse` command with a URL for which there is no
available spider, an exception is no longer raised (:issue:`3264`,
:issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
order mark`_ when determining the text encoding of the response body,
following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
.. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
.. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
- MIME sniffing takes the response body into account in FTP and HTTP/1.0
requests, as well as in cached requests (:issue:`4873`).
- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
is missing (:issue:`4873`).
- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
that does not match the asyncio event loop actually installed
(:issue:`5529`).
- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
returning only the last header (:issue:`5515`, :issue:`5526`).
- Fixed :class:`LinkExtractor
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
:issue:`4066`)
Documentation
~~~~~~~~~~~~~
- Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
(:issue:`5602`, :issue:`5608`).
- To enable
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
to do `brotli compression`_, installing brotli_ is now recommended instead
of installing brotlipy_, as the former provides a more recent version of
brotli.
.. _brotli: https://github.com/google/brotli
.. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
- :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
:issue:`5358`).
- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
(:issue:`3582`, :issue:`5432`).
.. _Common Crawl: https://commoncrawl.org/
.. _Google cache: http://www.googleguide.com/cached_pages.html
- The new :ref:`topics-components` topic covers enforcing requirements on
Scrapy components, like :ref:`downloader middlewares
<topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
:ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
<topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
has also been added (:issue:`4978`).
- :ref:`topics-settings` now indicates that setting values must be
:ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
- Removed outdated documentation (:issue:`5446`, :issue:`5373`,
:issue:`5369`, :issue:`5370`, :issue:`5554`).
- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
:issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
:issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
Quality assurance
~~~~~~~~~~~~~~~~~
- Added a continuous integration job to run `twine check`_ (:issue:`5655`,
:issue:`5656`).
.. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
:issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
:issue:`5671`, :issue:`5675`).
- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
:issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
- Applied minor code improvements (:issue:`5661`).
.. _release-2.6.3:
Scrapy 2.6.3 (2022-09-27)
-------------------------
- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
(:issue:`5634`, :issue:`5635`, :issue:`5636`).
- Upgraded the minimum versions of the following dependencies:
- cryptography_: 2.0 → 3.3
- pyOpenSSL_: 16.2.0 → 21.0.0
- service_identity_: 16.0.0 → 18.1.0
- Twisted_: 17.9.0 → 18.9.0
- zope.interface_: 4.1.3 → 5.0.0
(:issue:`5621`, :issue:`5632`)
- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
:issue:`5631`).
.. _release-2.6.2:
Scrapy 2.6.2 (2022-07-25)
-------------------------
**Security bug fix:**
- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
different :reqmeta:`proxy` metadata every time they process a request.
Because of request retries and redirects, the same request can be processed
by downloader middlewares more than once, including both
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
any third-party proxy-rotation downloader middleware.
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
To prevent the unintended leaking of proxy credentials, the behavior of
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
as follows when processing a request:
- If the request being processed defines :reqmeta:`proxy` metadata that
includes credentials, the ``Proxy-Authorization`` header is always
updated to feature those credentials.
- If the request being processed defines :reqmeta:`proxy` metadata
without credentials, the ``Proxy-Authorization`` header is removed
*unless* it was originally defined for the same proxy URL.
To remove proxy credentials while keeping the same proxy URL, remove
the ``Proxy-Authorization`` header.
- If the request has no :reqmeta:`proxy` metadata, or that metadata is a
falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
removed.
It is no longer possible to set a proxy URL through the
:reqmeta:`proxy` metadata but set the credentials through the
``Proxy-Authorization`` header. Set proxy credentials through the
:reqmeta:`proxy` metadata instead.
Also fixes the following regressions introduced in 2.6.0:
- :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
spiders (:issue:`5435`, :issue:`5436`)
- Installing a Twisted reactor before Scrapy does (e.g. importing
:mod:`twisted.internet.reactor` somewhere at the module level) no longer
prevents Scrapy from starting, as long as a different reactor is not
specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`)
- Fixed an exception that was being logged after the spider finished under
certain conditions (:issue:`5437`, :issue:`5440`)
- The ``--output``/``-o`` command-line parameter supports again a value
starting with a hyphen (:issue:`5444`, :issue:`5445`)
- The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`,
:issue:`5482`)
.. _release-2.6.1:
Scrapy 2.6.1 (2022-03-01)
@ -113,6 +452,9 @@ Backward-incompatible changes
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
other exceptions would be raised at run time. (:issue:`3559`)
- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects
is now named ``encoding``, in line with all other fields (:issue:`5130`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
@ -1897,6 +2239,59 @@ affect subclasses:
(:issue:`3884`)
.. _release-1.8.3:
Scrapy 1.8.3 (2022-07-25)
-------------------------
**Security bug fix:**
- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
processes a request with :reqmeta:`proxy` metadata, and that
:reqmeta:`proxy` metadata includes proxy credentials,
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
the ``Proxy-Authorization`` header, but only if that header is not already
set.
There are third-party proxy-rotation downloader middlewares that set
different :reqmeta:`proxy` metadata every time they process a request.
Because of request retries and redirects, the same request can be processed
by downloader middlewares more than once, including both
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
any third-party proxy-rotation downloader middleware.
These third-party proxy-rotation downloader middlewares could change the
:reqmeta:`proxy` metadata of a request to a new value, but fail to remove
the ``Proxy-Authorization`` header from the previous value of the
:reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
to a different proxy.
To prevent the unintended leaking of proxy credentials, the behavior of
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
as follows when processing a request:
- If the request being processed defines :reqmeta:`proxy` metadata that
includes credentials, the ``Proxy-Authorization`` header is always
updated to feature those credentials.
- If the request being processed defines :reqmeta:`proxy` metadata
without credentials, the ``Proxy-Authorization`` header is removed
*unless* it was originally defined for the same proxy URL.
To remove proxy credentials while keeping the same proxy URL, remove
the ``Proxy-Authorization`` header.
- If the request has no :reqmeta:`proxy` metadata, or that metadata is a
falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
removed.
It is no longer possible to set a proxy URL through the
:reqmeta:`proxy` metadata but set the credentials through the
``Proxy-Authorization`` header. Set proxy credentials through the
:reqmeta:`proxy` metadata instead.
.. _release-1.8.2:
Scrapy 1.8.2 (2022-03-01)
@ -2985,7 +3380,7 @@ New Features
~~~~~~~~~~~~
- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
- Support `brotli`_-compressed content; requires optional `brotlipy`_
- Support `brotli-compressed`_ content; requires optional `brotlipy`_
(:issue:`2535`)
- New :ref:`response.follow <response-follow-example>` shortcut
for creating requests (:issue:`1940`)
@ -3022,7 +3417,7 @@ New Features
- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
(:issue:`2740`)
.. _brotli: https://github.com/google/brotli
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://github.com/python-hyper/brotlipy/
Bug fixes

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx-rtd-theme>=0.5.2
sphinx==5.0.2
sphinx-hoverxref==1.1.1
sphinx-notfound-page==0.8
sphinx-rtd-theme==1.0.0

24
docs/topics/asyncio.rst
View File

@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this:
down to Scrapy 2.0 (earlier versions do not support
:mod:`asyncio`), you can copy the implementation of these functions
into your own code.
.. _enforce-asyncio-requirement:
Enforcing asyncio as a requirement
==================================
If you are writing a :ref:`component <topics-components>` that requires asyncio
to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
example::
from scrapy.utils.reactor import is_asyncio_reactor_installed
class MyComponent:
def __init__(self):
if not is_asyncio_reactor_installed():
raise ValueError(
f"{MyComponent.__qualname__} requires the asyncio Twisted "
f"reactor. Make sure you have it configured in the "
f"TWISTED_REACTOR setting. See the asyncio documentation "
f"of Scrapy for more information."
)

2
docs/topics/broad-crawls.rst
View File

@ -68,7 +68,7 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU and memory you crawler will have
to increase it will depend on how much CPU and memory your crawler will have
available.
A good starting point is ``100``::

20
docs/topics/commands.rst
View File

@ -271,11 +271,31 @@ crawl
Start crawling using a spider.
Supported options:
* ``-h, --help``: show a help message and exit
* ``-a NAME=VALUE``: set a spider argument (may be repeated)
* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
Usage examples::
$ scrapy crawl myspider
[ ... myspider starts crawling ... ]
$ scrapy -o myfile:csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
$ scrapy -O myfile:json myspider
[ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
$ scrapy -o myfile -t csv myspider
[ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
.. command:: check

84
docs/topics/components.rst Normal file
View File

@ -0,0 +1,84 @@
.. _topics-components:
==========
Components
==========
A Scrapy component is any class whose objects are created using
:func:`scrapy.utils.misc.create_instance`.
That includes the classes that you may assign to the following settings:
- :setting:`DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to
modify their behavior.
.. _enforce-component-requirements:
Enforcing component requirements
================================
Sometimes, your components may only be intended to work under certain
conditions. For example, they may require a minimum version of Scrapy to work as
intended, or they may require certain settings to have specific values.
In addition to describing those conditions in the documentation of your
component, it is a good practice to raise an exception from the ``__init__``
method of your component if those conditions are not met at run time.
In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
:ref:`extensions <topics-extensions>`, :ref:`item pipelines
<topics-item-pipeline>`, and :ref:`spider middlewares
<topics-spider-middleware>`, you should raise
:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
parameter to the exception so that it is printed in the logs, for the user to
see. For other components, feel free to raise whatever other exception feels
right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
version mismatch, while :exc:`ValueError` may be better if the issue is the
value of a setting.
If your requirement is a minimum Scrapy version, you may use
:attr:`scrapy.__version__` to enforce your requirement. For example::
from pkg_resources import parse_version
import scrapy
class MyComponent:
def __init__(self):
if parse_version(scrapy.__version__) < parse_version('2.7'):
raise RuntimeError(
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
f"later, which allow defining the process_spider_output "
f"method of spider middlewares as an asynchronous "
f"generator."
)

2
docs/topics/contracts.rst
View File

@ -102,7 +102,7 @@ override three methods:
.. method:: Contract.post_process(output)
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
converted to lists before being passed to this hook.
Raise :class:`~scrapy.exceptions.ContractFail` from
:class:`~scrapy.contracts.Contract.pre_process` or

133
docs/topics/coroutines.rst
View File

@ -19,14 +19,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.Request` callbacks.
.. note:: The callback output is not processed until the whole callback
finishes.
If you are using any custom or third-party :ref:`spider middleware
<topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
As a side effect, if the callback raises an exception, none of its
output is processed.
This is a known caveat of the current implementation that we aim to
address in a future version of Scrapy.
.. versionchanged:: 2.7
Output of async callbacks is now processed asynchronously instead of
collecting all of it first.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -41,12 +39,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
Usage
=====
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>`.
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
It must be defined as an :term:`asynchronous generator`. The input
``result`` parameter is an :term:`asynchronous iterable`.
See also :ref:`sync-async-spider-middleware` and
:ref:`universal-spider-middleware`.
.. versionadded:: 2.7
General usage
=============
There are several use cases for coroutines in Scrapy.
Code that would return Deferreds when written for previous Scrapy versions,
such as downloader middlewares and signal handlers, can be rewritten to be
shorter and cleaner::
from itemadapter import ItemAdapter
@ -106,7 +118,100 @@ Common use cases for asynchronous code include:
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares
======================================================
.. versionadded:: 2.7
The output of a :class:`~scrapy.Request` callback is passed as the ``result``
parameter to the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
of the first :ref:`spider middleware <topics-spider-middleware>` from the
:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
Then the output of that ``process_spider_output`` method is passed to the
``process_spider_output`` method of the next spider middleware, and so on for
every active spider middleware.
Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
in this chain of calls.
However, if any of the ``process_spider_output`` methods is defined as a
synchronous method, and the previous ``Request`` callback or
``process_spider_output`` method is a coroutine, there are some drawbacks to
the asynchronous-to-synchronous conversion that Scrapy does so that the
synchronous ``process_spider_output`` method gets a synchronous iterable as its
``result`` parameter:
- The whole output of the previous ``Request`` callback or
``process_spider_output`` method is awaited at this point.
- If an exception raises while awaiting the output of the previous
``Request`` callback or ``process_spider_output`` method, none of that
output will be processed.
This contrasts with the regular behavior, where all items yielded before
an exception raises are processed.
Asynchronous-to-synchronous conversions are supported for backward
compatibility, but they are deprecated and will stop working in a future
version of Scrapy.
To avoid asynchronous-to-synchronous conversions, when defining ``Request``
callbacks as coroutine methods or when using spider middlewares whose
``process_spider_output`` method is an :term:`asynchronous generator`, all
active spider middlewares must either have their ``process_spider_output``
method defined as an asynchronous generator or :ref:`define a
process_spider_output_async method <universal-spider-middleware>`.
.. note:: When using third-party spider middlewares that only define a
synchronous ``process_spider_output`` method, consider
:ref:`making them universal <universal-spider-middleware>` through
:ref:`subclassing <tut-inheritance>`.
.. _universal-spider-middleware:
Universal spider middlewares
============================
.. versionadded:: 2.7
To allow writing a spider middleware that supports asynchronous execution of
its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
while maintaining support for older Scrapy versions, you may define
``process_spider_output`` as a synchronous method and define an
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example::
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
# ... do something with r
yield r
async def process_spider_output_async(self, response, result, spider):
async for r in result:
# ... do something with r
yield r
.. note:: This is an interim measure to allow, for a time, to write code that
works in Scrapy 2.7 and later without requiring
asynchronous-to-synchronous conversions, and works in earlier Scrapy
versions as well.
In some future version of Scrapy, however, this feature will be
deprecated and, eventually, in a later version of Scrapy, this
feature will be removed, and all spider middlewares will be expected
to define their ``process_spider_output`` method as an asynchronous
generator.

28
docs/topics/debug.rst
View File

@ -150,3 +150,31 @@ available in all future runs should they be necessary again::
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
Visual Studio Code
==================
.. highlight:: json
To debug spiders with Visual Studio Code you can use the following ``launch.json``::
{
"version": "0.1.0",
"configurations": [
{
"name": "Python: Launch Scrapy Spider",
"type": "python",
"request": "launch",
"module": "scrapy",
"args": [
"runspider",
"${file}"
],
"console": "integratedTerminal"
}
]
}
Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in
your Scrapy spider.

4
docs/topics/downloader-middleware.rst
View File

@ -955,7 +955,7 @@ default because HTTP specs say so.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
---------------------
^^^^^^^^^^^^^^^^^^^^^
Default: ``-1``
@ -1119,7 +1119,7 @@ In order to use this parser:
.. _support-for-new-robots-parser:
Implementing support for a new parser
-------------------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can implement support for a new robots.txt_ parser by subclassing
the abstract base class :class:`~scrapy.robotstxt.RobotParser` and

32
docs/topics/exporters.rst
View File

@ -117,7 +117,7 @@ after your custom code.
Example::
from scrapy.exporter import XmlItemExporter
from scrapy.exporters import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
@ -195,17 +195,25 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Fields to export, their order [1]_ and their output names.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Possible values are:
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
- ``None`` (all fields [2]_, default)
- A list of fields::
['field1', 'field2']
- A dict where keys are fields and values are output names::
{'field1': 'Field 1', 'field2': 'Field 2'}
.. [1] Not all exporters respect the specified field order.
.. [2] When using :ref:`item objects <item-types>` that do not expose
all their possible fields, exporters that do not support exporting
a different subset of fields per item will only export the fields
found in the first item exported.
.. attribute:: export_empty_fields
@ -297,8 +305,8 @@ CsvItemExporter
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
CSV columns, their order and their column names. The
:attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)

2
docs/topics/extensions.rst
View File

@ -17,7 +17,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
hypothetical extension to handle `Google Sitemaps`_ would use settings like
``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps

17
docs/topics/feed-exports.rst
View File

@ -58,7 +58,7 @@ CSV
- Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- To specify columns to export and their order use
- To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
A list of fields to export, optional.
Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
order and their output names. See :attr:`BaseItemExporter.fields_to_export
<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
.. setting:: FEED_EXPORT_INDENT

4
docs/topics/item-pipeline.rst
View File

@ -186,6 +186,7 @@ item.
::
import hashlib
from pathlib import Path
from urllib.parse import quote
import scrapy
@ -214,8 +215,7 @@ item.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
# Store filename in item.
adapter["screenshot_filename"] = filename

2
docs/topics/leaks.rst
View File

@ -154,7 +154,7 @@ Too many spiders?
If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
ignore a particular class (and all its subclasses). For
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider

9
docs/topics/media-pipeline.rst
View File

@ -156,7 +156,6 @@ By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.

47
docs/topics/request-response.rst
View File

@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the
REQUEST_FINGERPRINTER_CLASS
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: VERSION
.. versionadded:: 2.7
Default: :class:`scrapy.utils.request.RequestFingerprinter`
@ -409,54 +409,54 @@ import path.
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: VERSION
.. versionadded:: 2.7
Default: ``'PREVIOUS_VERSION'``
Default: ``'2.6'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'PREVIOUS_VERSION'`` (default)
- ``'2.6'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy PREVIOUS_VERSION and earlier versions.
Scrapy 2.6 and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'VERSION'``
- ``'2.7'``
This implementation was introduced in Scrapy VERSION to fix an issue of the
This implementation was introduced in Scrapy 2.7 to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
If you are using the default value (``'2.6'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
setting to a custom request fingerprinter class that implements the 2.6 request
fingerprinting algorithm and does not log this warning (
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidade the current
Changing the request fingerprinting algorithm would invalidate the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'PREVIOUS_VERSION'``).
the default value (``'2.6'``).
.. _PREVIOUS_VERSION-request-fingerprinter:
.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
@ -464,6 +464,8 @@ Writing your own request fingerprinter
A request fingerprinter is a class that must implement the following method:
.. currentmodule:: None
.. method:: fingerprint(self, request)
Return a :class:`bytes` object that uniquely identifies *request*.
@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method:
Additionally, it may also implement the following methods:
.. classmethod:: from_crawler(cls, crawler)
:noindex:
If present, this class method is called to create a request fingerprinter
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
@ -495,11 +498,13 @@ Additionally, it may also implement the following methods:
:class:`~scrapy.settings.Settings` object. It must return a new instance of
the request fingerprinter.
The ``fingerprint`` method of the default request fingerprinter,
.. currentmodule:: scrapy.http
The :meth:`fingerprint` method of the default request fingerprinter,
:class:`scrapy.utils.request.RequestFingerprinter`, uses
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
in your ``fingerprint`` method implementation:
common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
@ -519,7 +524,7 @@ account::
You can also write your own fingerprinting logic from scratch.
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
- Caching saves CPU by ensuring that fingerprints are calculated only once
@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`~scrapy.utils.request.fingerprint`. For example::
:func:`scrapy.utils.request.fingerprint`. For example::
from scrapy.utils.request import fingerprint
@ -564,8 +569,8 @@ when available, and then falls back to
return request.meta['fingerprint']
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
without using the deprecated ``'2.6'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter::

30
docs/topics/settings.rst
View File

@ -98,6 +98,10 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Compatibility with pickle
=========================
Setting values must be :ref:`picklable <pickle-picklable>`.
Import paths and classes
========================
@ -560,7 +564,6 @@ This setting must be one of these string values:
set this if you want the behavior of Scrapy<1.1
- ``'TLSv1.1'``: forces TLS version 1.1
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -649,8 +652,9 @@ per ip address instead of per domain.
.. _spider-download_delay-attribute:
You can also change this setting per spider by setting ``download_delay``
spider attribute.
.. note::
This delay can be set per spider using :attr:`download_delay` spider attribute.
.. setting:: DOWNLOAD_HANDLERS
@ -1634,9 +1638,15 @@ which raises :exc:`Exception`, becomes::
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will install the default reactor defined by Twisted for the
current platform. This is to maintain backward compatibility and avoid possible
problems caused by using a non-default reactor.
means that Scrapy will use the existing reactor if one is already installed, or
install the default reactor defined by Twisted for the current platform. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
.. versionchanged:: 2.7
The :command:`startproject` command now sets this setting to
``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
``settings.py`` file.
For additional information, see :doc:`core/howto/choosing-reactor`.
@ -1652,14 +1662,14 @@ Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs.
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
:setting:`DEPTH_LIMIT`.
Use ``0`` to allow URLs of any length.
The default value is copied from the `Microsoft Internet Explorer maximum URL
The default value is copied from the `Microsoft Internet Explorer maximum URL
length`_, even though this setting exists for different reasons.
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246

26
docs/topics/spider-middleware.rst
View File

@ -102,27 +102,47 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.Request` objects and :ref:`item object
:class:`~scrapy.Request` objects and :ref:`item objects
<topics-items>`.
.. versionchanged:: 2.7
This method may be defined as an :term:`asynchronous generator`, in
which case ``result`` is an :term:`asynchronous iterable`.
Consider defining this method as an :term:`asynchronous generator`,
which will be a requirement in a future version of Scrapy. However, if
you plan on sharing your spider middleware with other people, consider
either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
as a minimum requirement of your spider middleware, or :ref:`making
your spider middleware universal <universal-spider-middleware>` so that
it works with Scrapy versions earlier than Scrapy 2.7.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item object <topics-items>`
:ref:`item objects <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output_async(response, result, spider)
.. versionadded:: 2.7
If defined, this method must be an :term:`asynchronous generator`,
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
.. method:: process_spider_exception(response, exception, spider)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects.
If it returns ``None``, Scrapy will continue processing this exception,

7
docs/topics/spiders.rst
View File

@ -181,9 +181,10 @@ scrapy.Spider
scraped data and/or more URLs to follow. Other Requests callbacks have
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.Request` and/or :ref:`item objects
<topics-items>`.
This method, as well as any other Request callback, must return a
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
<topics-items>`, or ``None``.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`

11
docs/topics/webservice.rst
View File

@ -1,11 +0,0 @@
.. _topics-webservice:
===========
Web Service
===========
webservice has been moved into a separate project.
It is hosted at:
https://github.com/scrapy-plugins/scrapy-jsonrpc

12
docs/utils/linkfix.py Executable file → Normal file
View File

@ -13,6 +13,8 @@ Author: dufferzafar
"""
import re
import sys
from pathlib import Path
def main():
@ -27,11 +29,11 @@ def main():
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
sys.exit(1)
# For every line, fix the respective file
for line in output_lines:
@ -51,14 +53,12 @@ def main():
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
Path(_filename).write_text(_contents, encoding="utf-8")
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = Path(_filename).read_text(encoding="utf-8")
_contents = _contents.replace(match.group(3), match.group(4))
else:

34
pylintrc
View File

@ -9,29 +9,19 @@ disable=abstract-method,
arguments-renamed,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-dict-items,
consider-using-from-import,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
consider-using-with,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
disallowed-name,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
@ -49,25 +39,17 @@ disable=abstract-method,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-fstring-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
@ -78,10 +60,7 @@ disable=abstract-method,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
super-with-arguments,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
@ -93,30 +72,23 @@ disable=abstract-method,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-lambda,
unnecessary-dunder-call,
unnecessary-pass,
unreachable,
unspecified-encoding,
unsubscriptable-object,
unused-argument,
unused-import,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

5
pytest.ini
View File

@ -21,3 +21,8 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated
ignore:typing.re is deprecated
ignore:typing.io is deprecated

2
scrapy/VERSION
View File

@ -1 +1 @@
2.6.1
2.7.1

24
scrapy/cmdline.py
View File

@ -7,13 +7,22 @@ import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
class ScrapyArgumentParser(argparse.ArgumentParser):
def _parse_optional(self, arg_string):
# if starts with -: it means that is a parameter not a argument
if arg_string[:2] == '-:':
return None
return super()._parse_optional(arg_string)
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# scrapy.utils.spider.iter_spider_classes
@ -23,7 +32,7 @@ def _iter_command_classes(module_name):
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and not obj == ScrapyCommand
and obj not in (ScrapyCommand, BaseRunSpiderCommand)
):
yield obj
@ -69,7 +78,8 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
version = scrapy.__version__
if inproject:
print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n")
print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
else:
print(f"Scrapy {version} - no active project\n")
@ -131,10 +141,10 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
cmd.add_options(parser)

19
scrapy/commands/__init__.py
View File

@ -3,9 +3,11 @@ Base class for Scrapy commands
"""
import os
import argparse
from typing import Any, Dict
from pathlib import Path
from typing import Any, Dict, Optional
from twisted.python import failure
from scrapy.crawler import CrawlerProcess
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
@ -14,15 +16,15 @@ from scrapy.exceptions import UsageError
class ScrapyCommand:
requires_project = False
crawler_process = None
crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}
exitcode = 0
def __init__(self):
self.settings = None # set in scrapy.cmdline
def __init__(self) -> None:
self.settings: Any = None # set in scrapy.cmdline
def set_crawler(self, crawler):
if hasattr(self, '_crawler'):
@ -93,8 +95,7 @@ class ScrapyCommand:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8")
if opts.pdb:
failure.startDebugMode()
@ -115,9 +116,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
help="append scraped items to the end of FILE (use - for stdout),"
" to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
help="dump scraped items into FILE, overwriting any existing file,"
" to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")

53
scrapy/commands/genspider.py
View File

@ -2,8 +2,9 @@ import os
import shutil
import string
from pathlib import Path
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
@ -62,8 +63,7 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
with open(template_file, "r") as f:
print(f.read())
print(template_file.read_text(encoding="utf-8"))
return
if len(args) != 2:
raise UsageError()
@ -98,11 +98,11 @@ class Command(ScrapyCommand):
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = abspath(dirname(spiders_module.__file__))
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
spiders_dir = "."
spider_file = f"{join(spiders_dir, module)}.py"
spiders_dir = Path(".")
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
@ -110,27 +110,33 @@ class Command(ScrapyCommand):
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
def _find_template(self, template: str) -> Optional[Path]:
template_file = Path(self.templates_dir, f'{template}.tmpl')
if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
return None
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(f" {splitext(filename)[0]}")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == '.tmpl':
print(f" {file.stem}")
def _spider_exists(self, name):
def _spider_exists(self, name: str) -> bool:
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
path = Path(name + ".py")
if path.exists():
print(f"{path.resolve()} already exists")
return True
return False
assert (
self.crawler_process is not None
), "crawler_process must be set before calling run"
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError:
@ -143,17 +149,18 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
spiders_dir = Path(cast(str, spiders_module.__file__)).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
print(f"{path} already exists")
return True
return False
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'spiders'
)
))

57
scrapy/commands/parse.py
View File

@ -5,13 +5,13 @@ from typing import Dict
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)
@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand):
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %default]")
help="maximum depth for parsing requests [default: %(default)s]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@ -110,16 +110,19 @@ class Command(BaseRunSpiderCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
for x in spider_output:
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)
return items, requests
return items, requests, opts, depth, spider, callback
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):
if getattr(spider, 'rules', None):
@ -158,6 +161,25 @@ class Command(BaseRunSpiderCommand):
logger.error('No response downloaded for: %(url)s',
{'url': url})
def scraped_data(self, args):
items, requests, opts, depth, spider, callback = args
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
# memorize first request
@ -191,23 +213,10 @@ class Command(BaseRunSpiderCommand):
# parse items and requests
depth = response.meta['_depth']
items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
itemproc.process_item(item, spider)
self.add_items(depth, items)
self.add_requests(depth, requests)
scraped_data = items if opts.output else []
if depth < opts.depth:
for req in requests:
req.meta['_depth'] = depth + 1
req.meta['_callback'] = req.callback
req.callback = callback
scraped_data += requests
return scraped_data
d = self.run_callback(response, cb, cb_kwargs)
d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
d.addCallback(self.scraped_data)
return d
# update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:

28
scrapy/commands/runspider.py
View File

@ -1,25 +1,25 @@
import sys
import os
from os import PathLike
from pathlib import Path
from importlib import import_module
from types import ModuleType
from typing import Union
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext not in ('.py', '.pyw'):
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
dirname = str(abspath.parent)
sys.path = [dirname] + sys.path
try:
module = import_module(fname)
module = import_module(abspath.stem)
finally:
if dirname:
sys.path.pop(0)
sys.path.pop(0)
return module
@ -40,13 +40,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
filename = Path(args[0])
if not filename.exists():
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {filename!r}: {e}\n")
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")

42
scrapy/commands/startproject.py
View File

@ -2,7 +2,7 @@ import re
import os
import string
from importlib.util import find_spec
from os.path import join, exists, abspath
from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@ -54,7 +54,7 @@ class Command(ScrapyCommand):
return True
return False
def _copytree(self, src, dst):
def _copytree(self, src: Path, dst: Path):
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@ -64,19 +64,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005
"""
ignore = IGNORE
names = os.listdir(src)
names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names)
if not os.path.exists(dst):
os.makedirs(dst)
if not dst.exists():
dst.mkdir(parents=True)
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.isdir(srcname):
srcname = src / name
dstname = dst / name
if srcname.is_dir():
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
@ -90,36 +90,36 @@ class Command(ScrapyCommand):
raise UsageError()
project_name = args[0]
project_dir = args[0]
if len(args) == 2:
project_dir = args[1]
project_dir = Path(args[1])
else:
project_dir = Path(args[0])
if exists(join(project_dir, 'scrapy.cfg')):
if (project_dir / 'scrapy.cfg').exists():
self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
return
if not self._is_valid_name(project_name):
self.exitcode = 1
return
self._copytree(self.templates_dir, abspath(project_dir))
move(join(project_dir, 'module'), join(project_dir, project_name))
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / 'module', project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'project'
)
))

2
scrapy/core/downloader/contextfactory.py
View File

@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
which allows TLS protocol negotiation
'A TLS/SSL connection established with [this method] may
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):

5
scrapy/core/downloader/handlers/file.py
View File

@ -1,3 +1,5 @@
from pathlib import Path
from w3lib.url import file_uri_to_path
from scrapy.utils.decorators import defers
@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
with open(filepath, 'rb') as fo:
body = fo.read()
body = Path(filepath).read_bytes()
respcls = get_response_class(url=request.url, body=body)
return respcls(url=request.url, body=body)

3
scrapy/core/downloader/handlers/http10.py
View File

@ -33,5 +33,4 @@ class HTTP10DownloadHandler:
crawler=self._crawler,
)
return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
return reactor.connectTCP(host, port, factory)

46
scrapy/core/downloader/handlers/http11.py
View File

@ -3,7 +3,6 @@
import ipaddress
import logging
import re
import warnings
from contextlib import suppress
from io import BytesIO
from time import time
@ -22,12 +21,11 @@ from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.exceptions import StopDownload
from scrapy.http import Headers
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.response import get_response_class
logger = logging.getLogger(__name__)
@ -279,17 +277,7 @@ class ScrapyAgent:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Zyte Smart Proxy Manager, it doesn't require "
"this mode anymore, so you should update scrapy-crawlera "
"to scrapy-zyte-smartproxy and remove '?noconnect' "
"from the Zyte Smart Proxy Manager URL.",
ScrapyDeprecationWarning,
)
if scheme == b'https' and not omitConnectTunnel:
if scheme == b'https':
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
return self._TunnelingAgent(
@ -300,18 +288,15 @@ class ScrapyAgent:
bindAddress=bindaddress,
pool=self._pool,
)
else:
proxyScheme = proxyScheme or b'http'
proxyHost = to_bytes(proxyHost, encoding='ascii')
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
proxyScheme = proxyScheme or b'http'
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,
proxyURI=to_bytes(proxyURI, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
return self._Agent(
reactor=reactor,
@ -384,8 +369,7 @@ class ScrapyAgent:
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": request, "handler": handler.__qualname__})
txresponse._transport.stopProducing()
with suppress(AttributeError):
txresponse._transport._producer.loseConnection()
txresponse._transport.loseConnection()
return {
"txresponse": txresponse,
"body": b"",
@ -417,7 +401,7 @@ class ScrapyAgent:
logger.warning(warning_msg, warning_args)
txresponse._transport._producer.loseConnection()
txresponse._transport.loseConnection()
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
@ -543,7 +527,7 @@ class _ResponseReader(protocol.Protocol):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
self.transport.stopProducing()
self.transport._producer.loseConnection()
self.transport.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
@ -581,7 +565,7 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["dataloss"])
return
elif not self._fail_on_dataloss_warned:
if not self._fail_on_dataloss_warned:
logger.warning("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",

14
scrapy/core/downloader/handlers/http2.py
View File

@ -1,4 +1,3 @@
import warnings
from time import time
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
@ -69,19 +68,8 @@ class ScrapyH2Agent:
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is not "
"supported by the downloader handler. If you use Zyte "
"Smart Proxy Manager, it doesn't require this mode "
"anymore, so you should update scrapy-crawlera to "
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
"Zyte Smart Proxy Manager URL."
)
if scheme == b'https' and not omit_connect_tunnel:
if scheme == b'https':
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
return self._ProxyAgent(

3
scrapy/core/downloader/tls.py
View File

@ -7,11 +7,9 @@ from twisted.internet.ssl import AcceptableCiphers
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
@ -20,7 +18,6 @@ METHOD_TLSv12 = 'TLSv1.2'
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only

4
scrapy/core/downloader/webclient.py
View File

@ -1,8 +1,8 @@
import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
@ -185,7 +185,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
@param version: The HTTP version.
@type version: L{bytes}
@param status: The HTTP status code, an integer represented as a
bytestring.
bytestring.
@type status: L{bytes}
@param message: The HTTP status message.
@type message: L{bytes}

18
scrapy/core/engine.py
View File

@ -136,7 +136,9 @@ class ExecutionEngine:
self.paused = False
def _next_request(self) -> None:
assert self.slot is not None # typing
if self.slot is None:
return
assert self.spider is not None # typing
if self.paused:
@ -184,7 +186,8 @@ class ExecutionEngine:
d.addErrback(lambda f: logger.info('Error while removing request from slot',
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider}))
d.addBoth(lambda _: self.slot.nextcall.schedule())
slot = self.slot
d.addBoth(lambda _: slot.nextcall.schedule())
d.addErrback(lambda f: logger.info('Error while scheduling new request',
exc_info=failure_to_exc_info(f),
extra={'spider': self.spider}))
@ -254,9 +257,7 @@ class ExecutionEngine:
def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
if spider is None:
spider = self.spider
else:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
category=ScrapyDeprecationWarning,
@ -264,7 +265,7 @@ class ExecutionEngine:
)
if spider is not self.spider:
logger.warning("The spider '%s' does not match the open spider", spider.name)
if spider is None:
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
return self._download(request, spider).addBoth(self._downloaded, request, spider)
@ -275,11 +276,14 @@ class ExecutionEngine:
self.slot.remove_request(request)
return self.download(result, spider) if isinstance(result, Request) else result
def _download(self, request: Request, spider: Spider) -> Deferred:
def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
assert self.slot is not None # typing
self.slot.add_request(request)
if spider is None:
spider = self.spider
def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
if not isinstance(result, (Response, Request)):
raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")

2
scrapy/core/http2/agent.py
View File

@ -140,7 +140,7 @@ class ScrapyProxyH2Agent(H2Agent):
connect_timeout: Optional[float] = None,
bind_address: Optional[bytes] = None,
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
super().__init__(
reactor=reactor,
pool=pool,
context_factory=context_factory,

2
scrapy/core/http2/protocol.py
View File

@ -42,7 +42,7 @@ class InvalidNegotiatedProtocol(H2Error):
self.negotiated_protocol = negotiated_protocol
def __str__(self) -> str:
return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}"
class RemoteTerminatedConnection(H2Error):

4
scrapy/core/http2/stream.py
View File

@ -151,11 +151,9 @@ class Stream:
self._deferred_response = Deferred(_cancel)
def __str__(self) -> str:
def __repr__(self) -> str:
return f'Stream(id={self.stream_id!r})'
__repr__ = __str__
@property
def _log_warnsize(self) -> bool:
"""Checks if we have received data which exceeds the download warnsize

19
scrapy/core/scheduler.py
View File

@ -1,8 +1,7 @@
import json
import logging
import os
from abc import abstractmethod
from os.path import exists, join
from pathlib import Path
from typing import Optional, Type, TypeVar
from twisted.internet.defer import Deferred
@ -324,19 +323,19 @@ class Scheduler(BaseScheduler):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
if jobdir is not None:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
dqdir = Path(jobdir, 'requests.queue')
if not dqdir.exists():
dqdir.mkdir(parents=True)
return str(dqdir)
return None
def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json')
if not exists(path):
path = Path(dqdir, 'active.json')
if not path.exists():
return []
with open(path) as f:
with path.open(encoding="utf-8") as f:
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f:
with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f:
json.dump(state, f)

78
scrapy/core/scraper.py
View File

@ -1,9 +1,22 @@
"""This module implements the Scraper component which parses responses and
extracts information from them"""
from __future__ import annotations
import logging
from collections import deque
from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
from typing import (
Any,
AsyncGenerator,
AsyncIterable,
Deque,
Generator,
Iterable,
Optional,
Set,
TYPE_CHECKING,
Tuple,
Union,
)
from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks
@ -13,12 +26,24 @@ from scrapy import signals, Spider
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
from scrapy.utils.defer import (
aiter_errback,
defer_fail,
defer_succeed,
iter_errback,
parallel,
parallel_async,
)
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from scrapy.crawler import Crawler
QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
@ -68,7 +93,7 @@ class Slot:
class Scraper:
def __init__(self, crawler):
def __init__(self, crawler: Crawler) -> None:
self.slot: Optional[Slot] = None
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
@ -145,9 +170,9 @@ class Scraper:
"""
if isinstance(result, Response):
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
else: # result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
# else result is a Failure
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
if isinstance(result, Response):
@ -167,6 +192,7 @@ class Scraper:
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
exc = _failure.value
if isinstance(exc, CloseSpider):
assert self.crawler.engine is not None # typing
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
return
logkws = self.logformatter.spider_error(_failure, request, response, spider)
@ -185,12 +211,19 @@ class Scraper:
spider=spider
)
def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
response: Response, spider: Spider) -> Deferred:
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
it: Union[Generator, AsyncGenerator]
if isinstance(result, AsyncIterable):
it = aiter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
else:
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
return dfd
def _process_spidermw_output(self, output: Any, request: Request, response: Response,
@ -200,6 +233,7 @@ class Scraper:
"""
assert self.slot is not None # typing
if isinstance(output, Request):
assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
elif is_item(output):
self.slot.itemproc_size += 1
@ -262,17 +296,15 @@ class Scraper:
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
logkws = self.logformatter.scraped(output, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)

193
scrapy/core/spidermw.py
View File

@ -3,32 +3,42 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
import logging
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import Any, Callable, Generator, Iterable, Union, cast
from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
from twisted.internet.defer import Deferred
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.python import MutableChain
from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
def _isiterable(o) -> bool:
return isinstance(o, Iterable)
return isinstance(o, (Iterable, AsyncIterable))
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
def __init__(self, *middlewares):
super().__init__(*middlewares)
self.downgrade_warning_done = False
@classmethod
def _get_mwlist_from_settings(cls, settings):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@ -39,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
process_spider_output = getattr(mw, 'process_spider_output', None)
process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
self.methods['process_spider_output'].appendleft(process_spider_output)
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
@ -51,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
try:
result = method(response=response, spider=spider)
if result is not None:
msg = (f"Middleware {method.__qualname__} must return None "
msg = (f"{method.__qualname__} must return None "
f"or raise an exception, got {type(result)}")
raise _InvalidOutput(msg)
except _InvalidOutput:
@ -60,17 +70,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
exception_processor_index: int, recover_to: MutableChain) -> Generator:
try:
for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex),
exception_processor_index)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable):
try:
for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex),
exception_processor_index)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterable):
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = self._process_spider_exception(response, spider, Failure(ex),
exception_processor_index)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterable):
return process_async(iterable)
return process_sync(iterable)
def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
start_index: int = 0) -> Union[Failure, MutableChain]:
@ -82,30 +110,82 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
method = cast(Callable, method)
result = method(response=response, exception=exception, spider=spider)
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
return self._process_spider_output(response, spider, result, method_index + 1)
dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return dfd.result
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
raise _InvalidOutput(msg)
elif result is None:
continue
else:
msg = (f"Middleware {method.__qualname__} must return None "
msg = (f"{method.__qualname__} must return None "
f"or an iterable, got {type(result)}")
raise _InvalidOutput(msg)
return _failure
# This method cannot be made async def, as _process_spider_exception relies on the Deferred result
# being available immediately which doesn't work when it's a wrapped coroutine.
# It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
@inlineCallbacks
def _process_spider_output(self, response: Response, spider: Spider,
result: Iterable, start_index: int = 0) -> MutableChain:
result: Union[Iterable, AsyncIterable], start_index: int = 0
) -> Deferred:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered = MutableChain()
recovered: Union[MutableChain, MutableAsyncChain]
last_result_is_async = isinstance(result, AsyncIterable)
if last_result_is_async:
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
# There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
# 1. def foo. Sync iterables are passed as is, async ones are downgraded.
# 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
# 3. def foo + async def foo_async. Iterables are passed to the respective method.
# Storing methods and method tuples in the same list is weird but we should be able to roll this back
# when we drop this compatibility feature.
method_list = islice(self.methods['process_spider_output'], start_index, None)
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
for method_index, method_pair in enumerate(method_list, start=start_index):
if method_pair is None:
continue
need_upgrade = need_downgrade = False
if isinstance(method_pair, tuple):
# This tuple handling is only needed until _async compatibility methods are removed.
method_sync, method_async = method_pair
method = method_async if last_result_is_async else method_sync
else:
method = method_pair
if not last_result_is_async and isasyncgenfunction(method):
need_upgrade = True
elif last_result_is_async and not isasyncgenfunction(method):
need_downgrade = True
try:
if need_upgrade:
# Iterable -> AsyncIterable
result = as_async_generator(result)
elif need_downgrade:
if not self.downgrade_warning_done:
logger.warning(f"Async iterable passed to {method.__qualname__} "
f"was downgraded to a non-async one")
self.downgrade_warning_done = True
assert isinstance(result, AsyncIterable)
# AsyncIterable -> Iterable
result = yield deferred_from_coro(collect_asyncgen(result))
if isinstance(recovered, AsyncIterable):
recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
@ -116,28 +196,75 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
else:
msg = (f"Middleware {method.__qualname__} must return an "
f"iterable, got {type(result)}")
if iscoroutine(result):
result.close() # Silence warning about not awaiting
msg = (
f"{method.__qualname__} must be an asynchronous "
f"generator (i.e. use yield)"
)
else:
msg = (
f"{method.__qualname__} must return an iterable, got "
f"{type(result)}"
)
raise _InvalidOutput(msg)
last_result_is_async = isinstance(result, AsyncIterable)
return MutableChain(result, recovered)
if last_result_is_async:
return MutableAsyncChain(result, recovered)
return MutableChain(result, recovered) # type: ignore[arg-type]
def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
recovered = MutableChain()
async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
recovered: Union[MutableChain, MutableAsyncChain]
if isinstance(result, AsyncIterable):
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
result = self._evaluate_iterable(response, spider, result, 0, recovered)
return MutableChain(self._process_spider_output(response, spider, result), recovered)
result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
if isinstance(result, AsyncIterable):
return MutableAsyncChain(result, recovered)
if isinstance(recovered, AsyncIterable):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered) # type: ignore[arg-type]
def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Deferred:
def process_callback_output(result: Iterable) -> MutableChain:
return self._process_callback_output(response, spider, result)
async def process_callback_output(result: Union[Iterable, AsyncIterable]
) -> Union[MutableChain, MutableAsyncChain]:
return await self._process_callback_output(response, spider, result)
def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
return self._process_spider_exception(response, spider, _failure)
dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
return dfd
def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
return self._process_chain('process_start_requests', start_requests, spider)
# This method is only needed until _async compatibility methods are removed.
@staticmethod
def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
normal_method = getattr(mw, methodname, None)
methodname_async = methodname + "_async"
async_method = getattr(mw, methodname_async, None)
if not async_method:
return normal_method
if not normal_method:
logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
f"without {methodname}, skipping this method.")
return None
if not isasyncgenfunction(async_method):
logger.error(f"{async_method.__qualname__} is not "
f"an async generator function, skipping this method.")
return normal_method
if isasyncgenfunction(normal_method):
logger.error(f"{normal_method.__qualname__} is an async "
f"generator function while {methodname_async} exists, "
f"skipping both methods.")
return None
return normal_method, async_method

32
scrapy/crawler.py
View File

@ -1,7 +1,10 @@
from __future__ import annotations
import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING, Optional
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@ -31,7 +34,15 @@ from scrapy.utils.log import (
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import install_reactor, verify_installed_reactor
from scrapy.utils.reactor import (
install_reactor,
is_asyncio_reactor_installed,
verify_installed_asyncio_event_loop,
verify_installed_reactor,
)
if TYPE_CHECKING:
from scrapy.utils.request import RequestFingerprinter
logger = logging.getLogger(__name__)
@ -72,31 +83,33 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,
)
reactor_class = self.settings.get("TWISTED_REACTOR")
reactor_class = self.settings["TWISTED_REACTOR"]
event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
install_reactor(reactor_class, event_loop)
else:
from twisted.internet import default
default.install()
from twisted.internet import reactor # noqa: F401
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
self.engine: Optional[ExecutionEngine] = None
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
@ -297,6 +310,7 @@ class CrawlerProcess(CrawlerRunner):
super().__init__(settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
self._initialized_reactor = False
def _signal_shutdown(self, signum, _):
from twisted.internet import reactor
@ -317,7 +331,9 @@ class CrawlerProcess(CrawlerRunner):
def _create_crawler(self, spidercls):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings, init_reactor=True)
init_reactor = not self._initialized_reactor
self._initialized_reactor = True
return Crawler(spidercls, self.settings, init_reactor=init_reactor)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
"""

7
scrapy/downloadermiddlewares/cookies.py
View File

@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@ -104,8 +103,8 @@ class CookiesMiddleware:
for key in ("name", "value", "path", "domain"):
if cookie.get(key) is None:
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
return
continue
if isinstance(cookie[key], (bool, float, int, str)):
@ -129,7 +128,7 @@ class CookiesMiddleware:
"""
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
if isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies

9
scrapy/downloadermiddlewares/decompression.py
View File

@ -9,10 +9,19 @@ import tarfile
import zipfile
from io import BytesIO
from tempfile import mktemp
from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.response import get_response_class
warn(
'scrapy.downloadermiddlewares.decompression is deprecated',
ScrapyDeprecationWarning,
stacklevel=2,
)
logger = logging.getLogger(__name__)

1
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -8,7 +8,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.gz import gunzip
from scrapy.utils.response import get_response_class
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
try:

59
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -45,31 +45,40 @@ class HttpProxyMiddleware:
return creds, proxy_url
def process_request(self, request, spider):
# ignore if proxy is already set
creds, proxy_url = None, None
if 'proxy' in request.meta:
if request.meta['proxy'] is None:
return
# extract credentials if present
creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
if request.meta['proxy'] is not None:
creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
elif self.proxies:
parsed = urlparse_cached(request)
scheme = parsed.scheme
if (
(
# 'no_proxy' is only supported by http schemes
scheme not in ('http', 'https')
or not proxy_bypass(parsed.hostname)
)
and scheme in self.proxies
):
creds, proxy_url = self.proxies[scheme]
self._set_proxy_and_creds(request, proxy_url, creds)
def _set_proxy_and_creds(self, request, proxy_url, creds):
if proxy_url:
request.meta['proxy'] = proxy_url
if creds and not request.headers.get('Proxy-Authorization'):
request.headers['Proxy-Authorization'] = b'Basic ' + creds
return
elif not self.proxies:
return
parsed = urlparse_cached(request)
scheme = parsed.scheme
# 'no_proxy' is only supported by http schemes
if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
return
if scheme in self.proxies:
self._set_proxy(request, scheme)
def _set_proxy(self, request, scheme):
creds, proxy = self.proxies[scheme]
request.meta['proxy'] = proxy
elif request.meta.get('proxy') is not None:
request.meta['proxy'] = None
if creds:
request.headers['Proxy-Authorization'] = b'Basic ' + creds
request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
request.meta['_auth_proxy'] = proxy_url
elif '_auth_proxy' in request.meta:
if proxy_url != request.meta['_auth_proxy']:
if b'Proxy-Authorization' in request.headers:
del request.headers[b'Proxy-Authorization']
del request.meta['_auth_proxy']
elif b'Proxy-Authorization' in request.headers:
if proxy_url:
request.meta['_auth_proxy'] = proxy_url
else:
del request.headers[b'Proxy-Authorization']

8
scrapy/downloadermiddlewares/redirect.py
View File

@ -8,7 +8,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
@ -56,10 +55,9 @@ class BaseRedirectMiddleware:
{'reason': reason, 'redirected': redirected, 'request': request},
extra={'spider': spider})
return redirected
else:
logger.debug("Discarding %(request)s: max redirections reached",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest("max redirections reached")
logger.debug("Discarding %(request)s: max redirections reached",
{'request': request}, extra={'spider': spider})
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
redirect_request = _build_redirect_request(

17
scrapy/downloadermiddlewares/retry.py
View File

@ -113,15 +113,14 @@ def get_retry_request(
stats.inc_value(f'{stats_base_key}/count')
stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
return new_request
else:
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None
class RetryMiddleware:

3
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -81,8 +81,7 @@ class RobotsTxtMiddleware:
return result
self._parsers[netloc].addCallback(cb)
return d
else:
return self._parsers[netloc]
return self._parsers[netloc]
def _logerror(self, failure, request, spider):
if failure.type is not IgnoreRequest:

4
scrapy/downloadermiddlewares/stats.py
View File

@ -1,9 +1,9 @@
from twisted.web import http
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
from twisted.web import http
def get_header_size(headers):
size = 0

4
scrapy/dupefilters.py
View File

@ -1,5 +1,5 @@
import logging
import os
from pathlib import Path
from typing import Optional, Set, Type, TypeVar
from warnings import warn
@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
self.file = open(os.path.join(path, 'requests.seen'), 'a+')
self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8")
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)

31
scrapy/exporters.py
View File

@ -8,6 +8,7 @@ import marshal
import pickle
import pprint
import warnings
from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
@ -68,6 +69,14 @@ class BaseItemExporter:
field_iter = item.field_names()
else:
field_iter = item.keys()
elif isinstance(self.fields_to_export, Mapping):
if include_empty:
field_iter = self.fields_to_export.items()
else:
field_iter = (
(x, y) for x, y in self.fields_to_export.items()
if x in item
)
else:
if include_empty:
field_iter = self.fields_to_export
@ -75,13 +84,17 @@ class BaseItemExporter:
field_iter = (x for x in self.fields_to_export if x in item)
for field_name in field_iter:
if field_name in item:
field_meta = item.get_field_meta(field_name)
value = self.serialize_field(field_meta, field_name, item[field_name])
if isinstance(field_name, str):
item_field, output_field = field_name, field_name
else:
item_field, output_field = field_name
if item_field in item:
field_meta = item.get_field_meta(item_field)
value = self.serialize_field(field_meta, output_field, item[item_field])
else:
value = default_value
yield field_name, value
yield output_field, value
class JsonLinesItemExporter(BaseItemExporter):
@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
if not self.fields_to_export:
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
row = list(self._build_row(self.fields_to_export))
if isinstance(self.fields_to_export, Mapping):
fields = self.fields_to_export.values()
else:
fields = self.fields_to_export
row = list(self._build_row(fields))
self.csv_writer.writerow(row)
@ -317,9 +334,9 @@ class PythonItemExporter(BaseItemExporter):
def _serialize_value(self, value):
if isinstance(value, Item):
return self.export_item(value)
elif is_item(value):
if is_item(value):
return dict(self._serialize_item(value))
elif is_listlike(value):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (str, bytes)):

16
scrapy/extensions/feedexport.py
View File

@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst
"""
import logging
import os
import re
import sys
import warnings
from datetime import datetime
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Callable, Optional, Tuple, Union
from typing import IO, Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
@ -101,7 +101,7 @@ class BlockingFeedStorage:
def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR']
if path and not os.path.isdir(path):
if path and not Path(path).is_dir():
raise OSError('Not a Directory: ' + str(path))
return NamedTemporaryFile(prefix='feed-', dir=path)
@ -141,11 +141,11 @@ class FileFeedStorage:
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, self.write_mode)
def open(self, spider) -> IO[Any]:
dirname = Path(self.path).parent
if dirname and not dirname.exists():
dirname.mkdir(parents=True)
return Path(self.path).open(self.write_mode)
def store(self, file):
file.close()

68
scrapy/extensions/httpcache.py
View File

@ -1,21 +1,22 @@
import gzip
import logging
import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.response import get_response_class
logger = logging.getLogger(__name__)
@ -79,25 +80,24 @@ class RFC2616Policy:
if b'no-store' in cc:
return False
# Never cache 304 (Not Modified) responses
elif response.status == 304:
if response.status == 304:
return False
# Cache unconditionally if configured to do so
elif self.always_store:
if self.always_store:
return True
# Any hint on response expiration is good
elif b'max-age' in cc or b'Expires' in response.headers:
if b'max-age' in cc or b'Expires' in response.headers:
return True
# Firefox fallbacks this statuses to one year expiration if none is set
elif response.status in (300, 301, 308):
if response.status in (300, 301, 308):
return True
# Other statuses without expiration requires at least one validator
elif response.status in (200, 203, 401):
if response.status in (200, 203, 401):
return b'Last-Modified' in response.headers or b'ETag' in response.headers
# Any other is probably not eligible for caching
# Makes no sense to cache responses that does not contain expiration
# info and can not be revalidated
else:
return False
return False
def is_cached_response_fresh(self, cachedresponse, request):
cc = self._parse_cachecontrol(cachedresponse)
@ -221,9 +221,9 @@ class DbmCacheStorage:
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.db = None
def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
def open_spider(self, spider: Spider):
dbpath = Path(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(str(dbpath), 'c')
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
@ -277,7 +277,7 @@ class FilesystemCacheStorage:
self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
def open_spider(self, spider: Spider):
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
@ -286,15 +286,15 @@ class FilesystemCacheStorage:
def close_spider(self, spider):
pass
def retrieve_response(self, spider, request):
def retrieve_response(self, spider: Spider, request: Request):
"""Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request)
if metadata is None:
return # not cached
rpath = self._get_request_path(spider, request)
with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:
rpath = Path(self._get_request_path(spider, request))
with self._open(rpath / 'response_body', 'rb') as f:
body = f.read()
with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:
with self._open(rpath / 'response_headers', 'rb') as f:
rawheaders = f.read()
url = metadata.get('response_url')
status = metadata['status']
@ -303,11 +303,11 @@ class FilesystemCacheStorage:
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
def store_response(self, spider: Spider, request: Request, response):
"""Store the given response in the cache."""
rpath = self._get_request_path(spider, request)
if not os.path.exists(rpath):
os.makedirs(rpath)
rpath = Path(self._get_request_path(spider, request))
if not rpath.exists():
rpath.mkdir(parents=True)
metadata = {
'url': request.url,
'method': request.method,
@ -315,29 +315,29 @@ class FilesystemCacheStorage:
'response_url': response.url,
'timestamp': time(),
}
with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
with self._open(rpath / 'meta', 'wb') as f:
f.write(to_bytes(repr(metadata)))
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
with self._open(rpath / 'pickled_meta', 'wb') as f:
pickle.dump(metadata, f, protocol=4)
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
with self._open(rpath / 'response_headers', 'wb') as f:
f.write(headers_dict_to_raw(response.headers))
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
with self._open(rpath / 'response_body', 'wb') as f:
f.write(response.body)
with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:
with self._open(rpath / 'request_headers', 'wb') as f:
f.write(headers_dict_to_raw(request.headers))
with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:
with self._open(rpath / 'request_body', 'wb') as f:
f.write(request.body)
def _get_request_path(self, spider, request):
def _get_request_path(self, spider: Spider, request: Request) -> str:
key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
return str(Path(self.cachedir, spider.name, key[0:2], key))
def _read_meta(self, spider, request):
rpath = self._get_request_path(spider, request)
metapath = os.path.join(rpath, 'pickled_meta')
if not os.path.exists(metapath):
def _read_meta(self, spider: Spider, request: Request):
rpath = Path(self._get_request_path(spider, request))
metapath = rpath / 'pickled_meta'
if not metapath.exists():
return # not found
mtime = os.stat(metapath).st_mtime
mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime:
return # expired
with self._open(metapath, 'rb') as f:

13
scrapy/extensions/memusage.py
View File

@ -75,15 +75,16 @@ class MemoryUsage:
self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
def _check_limit(self):
if self.get_virtual_size() > self.limit:
peak_mem_usage = self.get_virtual_size()
if peak_mem_usage > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}M at {socket.gethostname()}"
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/limit_notified', 1)
@ -92,6 +93,8 @@ class MemoryUsage:
self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
else:
self.crawler.stop()
else:
logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024})
def _check_warning(self):
if self.warned: # warn only once
@ -99,12 +102,12 @@ class MemoryUsage:
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value('memusage/warning_reached', 1)
mem = self.warning / 1024 / 1024
logger.warning("Memory usage reached %(memusage)dM",
logger.warning("Memory usage reached %(memusage)dMiB",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
subj = (
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}M at {socket.gethostname()}"
f"memory usage reached {mem}MiB at {socket.gethostname()}"
)
self._send_report(self.notify_mails, subj)
self.crawler.stats.set_value('memusage/warning_notified', 1)

12
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
import pickle
from pathlib import Path
from scrapy import signals
from scrapy.exceptions import NotConfigured
@ -25,16 +25,16 @@ class SpiderState:
def spider_closed(self, spider):
if self.jobdir:
with open(self.statefn, 'wb') as f:
with Path(self.statefn).open('wb') as f:
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
if self.jobdir and os.path.exists(self.statefn):
with open(self.statefn, 'rb') as f:
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open('rb') as f:
spider.state = pickle.load(f)
else:
spider.state = {}
@property
def statefn(self):
return os.path.join(self.jobdir, 'spider.state')
def statefn(self) -> str:
return str(Path(self.jobdir, 'spider.state'))

2
scrapy/extensions/telnet.py
View File

@ -11,6 +11,7 @@ import binascii
import os
from twisted.internet import protocol
try:
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
@ -26,7 +27,6 @@ from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
from scrapy.utils.decorators import defers
logger = logging.getLogger(__name__)
# signal to update telnet variables

2
scrapy/extensions/throttle.py
View File

@ -56,7 +56,7 @@ class AutoThrottle:
{
'slot': key, 'concurrency': conc,
'delay': slot.delay * 1000, 'delaydiff': diff * 1000,
'latency': latency * 1000, 'size': size
'latency': latency * 1000, 'size': size,
},
extra={'spider': spider}
)

17
scrapy/http/headers.py
View File

@ -1,3 +1,5 @@
from collections.abc import Mapping
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@ -10,6 +12,13 @@ class Headers(CaselessDict):
self.encoding = encoding
super().__init__(seq)
def update(self, seq):
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = {}
for k, v in seq:
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
def normkey(self, key):
"""Normalize key to bytes"""
return self._tobytes(key.title())
@ -28,12 +37,11 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
elif isinstance(x, str):
if isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
if isinstance(x, int):
return str(x).encode(self.encoding)
else:
raise TypeError(f'Unsupported value type: {type(x)}')
raise TypeError(f'Unsupported value type: {type(x)}')
def __getitem__(self, key):
try:
@ -86,4 +94,5 @@ class Headers(CaselessDict):
def __copy__(self):
return self.__class__(self)
copy = __copy__

6
scrapy/http/request/__init__.py
View File

@ -121,11 +121,9 @@ class Request(object_ref):
def encoding(self) -> str:
return self._encoding
def __str__(self) -> str:
def __repr__(self) -> str:
return f"<{self.method} {self.url}>"
__repr__ = __str__
def copy(self) -> "Request":
return self.replace()
@ -187,7 +185,7 @@ class Request(object_ref):
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
if type(self) is not Request:
if type(self) is not Request: # pylint: disable=unidiomatic-typecheck
d["_class"] = self.__module__ + '.' + self.__class__.__name__
return d

4
scrapy/http/request/form.py
View File

@ -189,7 +189,7 @@ def _select_value(ele: SelectElement, n: str, v: str):
# And for select tags without options
o = ele.value_options
return (n, o[0]) if o else (None, None)
elif v is not None and multiple:
if v is not None and multiple:
# This is a workround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath('.//option[@selected]')
@ -235,7 +235,7 @@ def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tup
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
elif len(el) > 1:
if len(el) > 1:
raise ValueError(f"Multiple elements found ({el!r}) matching the "
f"criteria in clickdata: {clickdata!r}")
else:

15
scrapy/http/response/__init__.py
View File

@ -32,7 +32,7 @@ class Response(object_ref):
def __init__(
self,
url,
url: str,
status=200,
headers=None,
body=b"",
@ -75,7 +75,7 @@ class Response(object_ref):
def _get_url(self):
return self._url
def _set_url(self, url):
def _set_url(self, url: str):
if isinstance(url, str):
self._url = url
else:
@ -100,11 +100,9 @@ class Response(object_ref):
body = property(_get_body, obsolete_setter(_set_body, 'body'))
def __str__(self):
def __repr__(self):
return f"<{self.status} {self.url}>"
__repr__ = __str__
def copy(self):
"""Return a copy of this Response"""
return self.replace()
@ -142,8 +140,7 @@ class Response(object_ref):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
# type: (...) -> Request
dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
@ -181,8 +178,8 @@ class Response(object_ref):
def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
# type: (...) -> Generator[Request, None, None]
dont_filter=False, errback=None, cb_kwargs=None,
flags=None) -> Generator[Request, None, None]:
"""
.. versionadded:: 2.0

20
scrapy/http/response/text.py
View File

@ -11,8 +11,13 @@ from typing import Generator, Tuple
from urllib.parse import urljoin
import parsel
from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
http_content_type_encoding, resolve_encoding)
from w3lib.encoding import (
html_body_declared_encoding,
html_to_unicode,
http_content_type_encoding,
resolve_encoding,
read_bom,
)
from w3lib.html import get_base_url, strip_html5_whitespace
from scrapy.http import Request
@ -62,6 +67,7 @@ class TextResponse(Response):
def _declared_encoding(self):
return (
self._encoding
or self._bom_encoding()
or self._headers_encoding()
or self._body_declared_encoding()
)
@ -130,6 +136,10 @@ class TextResponse(Response):
def _body_declared_encoding(self):
return html_body_declared_encoding(self.body)
@memoizemethod_noargs
def _bom_encoding(self):
return read_bom(self.body)[0]
@property
def selector(self):
from scrapy.selector import Selector
@ -145,8 +155,7 @@ class TextResponse(Response):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
# type: (...) -> Request
dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request:
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
@ -187,8 +196,7 @@ class TextResponse(Response):
def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
dont_filter=False, errback=None, cb_kwargs=None, flags=None,
css=None, xpath=None):
# type: (...) -> Generator[Request, None, None]
css=None, xpath=None) -> Generator[Request, None, None]:
"""
A generator that produces :class:`~.Request` instances to follow all
links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s

89
scrapy/linkextractors/__init__.py
View File

@ -6,18 +6,6 @@ This package contains a collection of Link Extractors.
For more info see docs/topics/link-extractors.rst
"""
import re
from urllib.parse import urlparse
from warnings import warn
from parsel.csstranslator import HTMLTranslator
from w3lib.url import canonicalize_url
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import (
url_is_from_any_domain, url_has_any_extension,
)
# common file extensions that are not followed if they occur in links
IGNORED_EXTENSIONS = [
@ -55,82 +43,5 @@ def _is_valid_url(url):
return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor:
_csstranslator = HTMLTranslator()
def __new__(cls, *args, **kwargs):
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__new__(cls)
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
self.link_extractor = link_extractor
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(allow)]
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(deny)]
self.allow_domains = set(arg_to_iter(allow_domains))
self.deny_domains = set(arg_to_iter(deny_domains))
self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,
arg_to_iter(restrict_css)))
self.canonicalize = canonicalize
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(restrict_text)]
def _link_allowed(self, link):
if not _is_valid_url(link.url):
return False
if self.allow_res and not _matches(link.url, self.allow_res):
return False
if self.deny_res and _matches(link.url, self.deny_res):
return False
parsed_url = urlparse(link.url)
if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):
return False
if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):
return False
if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
return False
if self.restrict_text and not _matches(link.text, self.restrict_text):
return False
return True
def matches(self, url):
if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
return False
if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
return False
allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]
denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
return any(allowed) and not any(denied)
def _process_links(self, links):
links = [x for x in links if self._link_allowed(x)]
if self.canonicalize:
for link in links:
link.url = canonicalize_url(link.url)
links = self.link_extractor._process_links(links)
return links
def _extract_links(self, *args, **kwargs):
return self.link_extractor._extract_links(*args, **kwargs)
# Top-level imports
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor

85
scrapy/linkextractors/lxmlhtml.py
View File

@ -3,17 +3,19 @@ Link extractor based on lxml.html
"""
import operator
from functools import partial
from urllib.parse import urljoin
from urllib.parse import urljoin, urlparse
import lxml.etree as etree
from lxml import etree
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url, safe_url_string
from scrapy.link import Link
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches,
_re_type, re)
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list
from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
# from lxml/src/lxml/html/__init__.py
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
@ -101,7 +103,8 @@ class LxmlParserLinkExtractor:
return links
class LxmlLinkExtractor(FilteringLinkExtractor):
class LxmlLinkExtractor:
_csstranslator = HTMLTranslator()
def __init__(
self,
@ -121,7 +124,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
restrict_text=None,
):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
lx = LxmlParserLinkExtractor(
self.link_extractor = LxmlParserLinkExtractor(
tag=partial(operator.contains, tags),
attr=partial(operator.contains, attrs),
unique=unique,
@ -129,18 +132,64 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
strip=strip,
canonicalized=canonicalize
)
super().__init__(
link_extractor=lx,
allow=allow,
deny=deny,
allow_domains=allow_domains,
deny_domains=deny_domains,
restrict_xpaths=restrict_xpaths,
restrict_css=restrict_css,
canonicalize=canonicalize,
deny_extensions=deny_extensions,
restrict_text=restrict_text,
)
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(allow)]
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(deny)]
self.allow_domains = set(arg_to_iter(allow_domains))
self.deny_domains = set(arg_to_iter(deny_domains))
self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath,
arg_to_iter(restrict_css)))
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
self.canonicalize = canonicalize
self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
for x in arg_to_iter(restrict_text)]
def _link_allowed(self, link):
if not _is_valid_url(link.url):
return False
if self.allow_res and not _matches(link.url, self.allow_res):
return False
if self.deny_res and _matches(link.url, self.deny_res):
return False
parsed_url = urlparse(link.url)
if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):
return False
if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):
return False
if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
return False
if self.restrict_text and not _matches(link.text, self.restrict_text):
return False
return True
def matches(self, url):
if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
return False
if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
return False
allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]
denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
return any(allowed) and not any(denied)
def _process_links(self, links):
links = [x for x in links if self._link_allowed(x)]
if self.canonicalize:
for link in links:
link.url = canonicalize_url(link.url)
links = self.link_extractor._process_links(links)
return links
def _extract_links(self, *args, **kwargs):
return self.link_extractor._extract_links(*args, **kwargs)
def extract_links(self, response):
"""Returns a list of :class:`~scrapy.link.Link` objects from the

9
scrapy/middleware.py
View File

@ -1,7 +1,7 @@
import logging
import pprint
from collections import defaultdict, deque
from typing import Callable, Deque, Dict, Optional, cast, Iterable
from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast
from twisted.internet.defer import Deferred
@ -19,10 +19,11 @@ class MiddlewareManager:
component_name = 'foo middleware'
def __init__(self, *middlewares):
def __init__(self, *middlewares: Any) -> None:
self.middlewares = middlewares
# Optional because process_spider_output and process_spider_exception can be None
self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
# Only process_spider_output and process_spider_exception can be None.
# Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)

2
scrapy/pipelines/__init__.py
View File

@ -18,7 +18,7 @@ class ItemPipelineManager(MiddlewareManager):
return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
def _add_middleware(self, pipe):
super(ItemPipelineManager, self)._add_middleware(pipe)
super()._add_middleware(pipe)
if hasattr(pipe, 'process_item'):
self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))

47
scrapy/pipelines/files.py
View File

@ -13,6 +13,8 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from pathlib import Path
from typing import DefaultDict, Optional, Set
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@ -39,41 +41,40 @@ class FileException(Exception):
class FSFilesStore:
def __init__(self, basedir):
def __init__(self, basedir: str):
if '://' in basedir:
basedir = basedir.split('://', 1)[1]
self.basedir = basedir
self._mkdir(self.basedir)
self.created_directories = defaultdict(set)
self._mkdir(Path(self.basedir))
self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
def persist_file(self, path, buf, info, meta=None, headers=None):
def persist_file(self, path: str, buf, info, meta=None, headers=None):
absolute_path = self._get_filesystem_path(path)
self._mkdir(os.path.dirname(absolute_path), info)
with open(absolute_path, 'wb') as f:
f.write(buf.getvalue())
self._mkdir(absolute_path.parent, info)
absolute_path.write_bytes(buf.getvalue())
def stat_file(self, path, info):
def stat_file(self, path: str, info):
absolute_path = self._get_filesystem_path(path)
try:
last_modified = os.path.getmtime(absolute_path)
last_modified = absolute_path.stat().st_mtime
except os.error:
return {}
with open(absolute_path, 'rb') as f:
with absolute_path.open('rb') as f:
checksum = md5sum(f)
return {'last_modified': last_modified, 'checksum': checksum}
def _get_filesystem_path(self, path):
def _get_filesystem_path(self, path: str) -> Path:
path_comps = path.split('/')
return os.path.join(self.basedir, *path_comps)
return Path(self.basedir, *path_comps)
def _mkdir(self, dirname, domain=None):
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
seen = self.created_directories[domain] if domain else set()
if dirname not in seen:
if not os.path.exists(dirname):
os.makedirs(dirname)
seen.add(dirname)
if str(dirname) not in seen:
if not dirname.exists():
dirname.mkdir(parents=True)
seen.add(str(dirname))
class S3FilesStore:
@ -220,16 +221,14 @@ class GCSFilesStore:
checksum = blob.md5_hash
last_modified = time.mktime(blob.updated.timetuple())
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
return {}
blob_path = self._get_blob_path(path)
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
return headers['Content-Type']
else:
return 'application/octet-stream'
return 'application/octet-stream'
def _get_blob_path(self, path):
return self.prefix + path
@ -374,8 +373,8 @@ class FilesPipeline(MediaPipeline):
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)
def _get_store(self, uri):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
def _get_store(self, uri: str):
if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
scheme = 'file'
else:
scheme = urlparse(uri).scheme
@ -510,7 +509,7 @@ class FilesPipeline(MediaPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1]
media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the
# mime type then extension or default to empty string otherwise
if media_ext not in mimetypes.types_map:

48
scrapy/pipelines/images.py
View File

@ -5,23 +5,28 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem, NotConfigured
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.pipelines.files import FileException, FilesPipeline
# TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.misc import md5sum
from scrapy.utils.python import to_bytes
from scrapy.utils.python import get_func_args, to_bytes
class NoimagesDrop(DropItem):
"""Product with no images exception"""
def __init__(self, *args, **kwargs):
warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
super().__init__(*args, **kwargs)
class ImageException(FileException):
"""General image error exception"""
@ -87,6 +92,8 @@ class ImagesPipeline(FilesPipeline):
resolve('IMAGES_THUMBS'), self.THUMBS
)
self._deprecated_convert_image = None
@classmethod
def from_settings(cls, settings):
s3store = cls.STORE_SCHEMES['s3']
@ -137,16 +144,34 @@ class ImagesPipeline(FilesPipeline):
f"({width}x{height} < "
f"{self.min_width}x{self.min_height})")
image, buf = self.convert_image(orig_image)
if self._deprecated_convert_image is None:
self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image)
if self._deprecated_convert_image:
warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, '
'overriden method does not accept response_body argument.',
category=ScrapyDeprecationWarning)
if self._deprecated_convert_image:
image, buf = self.convert_image(orig_image)
else:
image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body))
yield path, image, buf
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
thumb_image, thumb_buf = self.convert_image(image, size)
if self._deprecated_convert_image:
thumb_image, thumb_buf = self.convert_image(image, size)
else:
thumb_image, thumb_buf = self.convert_image(image, size, buf)
yield thumb_path, thumb_image, thumb_buf
def convert_image(self, image, size=None):
if image.format == 'PNG' and image.mode == 'RGBA':
def convert_image(self, image, size=None, response_body=None):
if response_body is None:
warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, '
'method called without response_body argument.',
category=ScrapyDeprecationWarning, stacklevel=2)
if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA':
background = self._Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
@ -160,7 +185,16 @@ class ImagesPipeline(FilesPipeline):
if size:
image = image.copy()
image.thumbnail(size, self._Image.ANTIALIAS)
try:
# Image.Resampling.LANCZOS was added in Pillow 9.1.0
# remove this try except block,
# when updating the minimum requirements for Pillow.
resampling_filter = self._Image.Resampling.LANCZOS
except AttributeError:
resampling_filter = self._Image.ANTIALIAS
image.thumbnail(size, resampling_filter)
elif response_body is not None and image.format == 'JPEG':
return image, response_body
buf = BytesIO()
image.save(buf, 'JPEG')

10
scrapy/responsetypes.py
View File

@ -50,11 +50,10 @@ class ResponseTypes:
"""Return the most appropriate Response class for the given mimetype"""
if mimetype is None:
return Response
elif mimetype in self.classes:
if mimetype in self.classes:
return self.classes[mimetype]
else:
basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response)
basetype = f"{mimetype.split('/')[0]}/*"
return self.classes.get(basetype, Response)
def from_content_type(self, content_type, content_encoding=None):
"""Return the most appropriate Response class from an HTTP Content-Type
@ -91,8 +90,7 @@ class ResponseTypes:
mimetype, encoding = self.mimetypes.guess_type(filename)
if mimetype and not encoding:
return self.from_mimetype(mimetype)
else:
return Response
return Response
def from_body(self, body):
"""Try to guess the appropriate response based on the body content.

1
scrapy/robotstxt.py
View File

@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)

66
scrapy/settings/__init__.py
View File

@ -24,8 +24,7 @@ def get_settings_priority(priority):
"""
if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
return priority
class SettingsAttribute:
@ -51,11 +50,9 @@ class SettingsAttribute:
self.value = value
self.priority = priority
def __str__(self):
def __repr__(self):
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
__repr__ = __str__
class BaseSettings(MutableMapping):
"""
@ -197,6 +194,38 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
def getdictorlist(self, name, default=None):
"""Get a setting value as either a :class:`dict` or a :class:`list`.
If the setting is already a dict or a list, a copy of it will be
returned.
If it is a string it will be evaluated as JSON, or as a comma-separated
list of strings as a fallback.
For example, settings populated from the command line will return:
- ``{'key1': 'value1', 'key2': 'value2'}`` if set to
``'{"key1": "value1", "key2": "value2"}'``
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
:param name: the setting name
:type name: string
:param default: the value to return if no setting is found
:type default: any
"""
value = self.get(name, default)
if value is None:
return {}
if isinstance(value, str):
try:
return json.loads(value)
except ValueError:
return value.split(',')
return copy.deepcopy(value)
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
@ -230,8 +259,7 @@ class BaseSettings(MutableMapping):
"""
if len(self) > 0:
return max(self.getpriority(name) for name in self)
else:
return get_settings_priority('default')
return get_settings_priority('default')
def __setitem__(self, name, value):
self.set(name, value)
@ -405,30 +433,6 @@ class BaseSettings(MutableMapping):
p.text(pformat(self.copy_to_dict()))
class _DictProxy(MutableMapping):
def __init__(self, settings, priority):
self.o = {}
self.settings = settings
self.priority = priority
def __len__(self):
return len(self.o)
def __getitem__(self, k):
return self.o[k]
def __setitem__(self, k, v):
self.settings.set(k, v, priority=self.priority)
self.o[k] = v
def __delitem__(self, k):
del self.o[k]
def __iter__(self, k, v):
return iter(self.o)
class Settings(BaseSettings):
"""
This object stores Scrapy settings for the configuration of internal

6
scrapy/settings/default_settings.py
View File

@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to:
import sys
from importlib import import_module
from os.path import join, abspath, dirname
from pathlib import Path
AJAXCRAWL_ENABLED = False
@ -248,7 +248,7 @@ REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
@ -288,7 +288,7 @@ STATS_DUMP = True
STATSMAILER_RCPTS = []
TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())
URLLENGTH_LIMIT = 2083

5
scrapy/shell.py
View File

@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
class Shell:
@ -76,6 +77,10 @@ class Shell:
banner=self.vars.pop('banner', ''))
def _schedule(self, request, spider):
if is_asyncio_reactor_installed():
# set the asyncio event loop for the current thread
event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
set_asyncio_event_loop(event_loop_path)
spider = self._open_spider(request, spider)
d = _request_deferred(request)
d.addCallback(lambda x: (x, spider))

Some files were not shown because too many files have changed in this diff Show More