RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into integrate-mime

This commit is contained in:
Akshay Sharma 2022-06-20 20:19:04 -04:00 committed by GitHub
parent b9effde875 d8223adfac
commit bedd4e51e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
165 changed files with 4012 additions and 1129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.bandit.yml
View File

@ -8,6 +8,7 @@ skips:
- B311
- B320
- B321
- B324
- B402 # https://github.com/scrapy/scrapy/issues/4180
- B403
- B404

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.5.0
current_version = 2.6.1
commit = True
tag = True
tag_name = {new_version}

12
.github/workflows/checks.yml vendored
View File

@ -8,10 +8,10 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.9
- python-version: "3.10"
env:
TOXENV: security
- python-version: 3.9
- python-version: "3.10"
env:
TOXENV: flake8
# Pylint requires installing reppy, which does not support Python 3.9
@ -19,11 +19,10 @@ jobs:
- python-version: 3.8
env:
TOXENV: pylint
TOX_PIP_VERSION: 20.3.3
- python-version: 3.9
- python-version: 3.7
env:
TOXENV: typing
- python-version: 3.8 # Keep in sync with .readthedocs.yml
- python-version: "3.10" # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
@ -38,8 +37,5 @@ jobs:
- name: Run check
env: ${{ matrix.env }}
run: |
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
pip install tox-pip-version
fi
pip install -U tox
tox

4
.github/workflows/publish.yml vendored
View File

@ -9,10 +9,10 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.9
python-version: "3.10"
- name: Check Tag
id: check-release-tag

2
.github/workflows/tests-macos.yml vendored
View File

@ -7,7 +7,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2

29
.github/workflows/tests-ubuntu.yml vendored
View File

@ -8,31 +8,34 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.7
env:
TOXENV: py
- python-version: 3.8
env:
TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: asyncio
- python-version: pypy3
env:
TOXENV: pypy3
PYPY_VERSION: 3.6-v7.3.3
PYPY_VERSION: 3.9-v7.3.9
# pinned deps
- python-version: 3.6.12
- python-version: 3.7.13
env:
TOXENV: pinned
- python-version: 3.6.12
- python-version: 3.7.13
env:
TOXENV: asyncio-pinned
- python-version: pypy3
env:
TOXENV: pypy3-pinned
PYPY_VERSION: 3.6-v7.2.0
PYPY_VERSION: 3.7-v7.3.5
# extras
# extra-deps includes reppy, which does not support Python 3.9
@ -40,15 +43,6 @@ jobs:
- python-version: 3.8
env:
TOXENV: extra-deps
TOX_PIP_VERSION: 20.3.3
# 3.10-pre
- python-version: "3.10.0-beta.4"
env:
TOXENV: py
- python-version: "3.10.0-beta.4"
env:
TOXENV: asyncio
steps:
- uses: actions/checkout@v2
@ -75,9 +69,6 @@ jobs:
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
pip install tox-pip-version
fi
pip install -U tox
tox

18
.github/workflows/tests-windows.yml vendored
View File

@ -8,19 +8,21 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: 3.6
env:
TOXENV: windows-pinned
- python-version: 3.7
env:
TOXENV: py
TOXENV: windows-pinned
- python-version: 3.8
env:
TOXENV: py
# https://twistedmatrix.com/trac/ticket/9990
#- python-version: 3.9
#env:
#TOXENV: py
- python-version: 3.9
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: asyncio
steps:
- uses: actions/checkout@v2

9
.readthedocs.yml
View File

@ -5,12 +5,13 @@ sphinx:
fail_on_warning: true
build:
image: latest
os: ubuntu-20.04
tools:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
python: "3.10" # Keep in sync with .github/workflows/checks.yml
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.8 # Keep in sync with .github/workflows/checks.yml
install:
- requirements: docs/requirements.txt
- path: .

4
README.rst
View File

@ -1,3 +1,5 @@
.. image:: https://scrapy.org/img/scrapylogo.png
======
Scrapy
======
@ -55,7 +57,7 @@ including a list of features.
Requirements
============
* Python 3.6+
* Python 3.7+
* Works on Linux, Windows, macOS, BSD
Install

6
conftest.py
View File

@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest):
pytest.skip('This test is only run with --reactor=asyncio')
@pytest.fixture(autouse=True)
def only_not_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
pytest.skip('This test is only run without --reactor=asyncio')
def pytest_configure(config):
if config.getoption("--reactor") == "asyncio":
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

2
docs/README.rst
View File

@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
Start over
----------
To cleanup all generated documentation files and start from scratch run::
To clean up all generated documentation files and start from scratch run::
make clean

31
docs/_static/selectors-sample1.html vendored
View File

@ -1,16 +1,17 @@
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
</div>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<base href='http://example.com/' />
<title>Example website</title>
</head>
<body>
<div id='images'>
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
</div>
</body>
</html>

7
docs/conf.py
View File

@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
# Never documented before, and deprecated now.
r'^scrapy\.item\.DictItem$',
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
# Implementation detail of LxmlLinkExtractor
@ -295,7 +294,9 @@ intersphinx_mapping = {
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
}
intersphinx_disabled_reftypes = []
# Options for sphinx-hoverxref options
@ -304,10 +305,14 @@ intersphinx_mapping = {
hoverxref_auto_ref = True
hoverxref_role_types = {
"class": "tooltip",
"command": "tooltip",
"confval": "tooltip",
"hoverxref": "tooltip",
"mod": "tooltip",
"ref": "tooltip",
"reqmeta": "tooltip",
"setting": "tooltip",
"signal": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

8
docs/conftest.py
View File

@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
from sybil.parsers.codeblock import CodeBlockParser
try:
# >2.0.1
from sybil.parsers.codeblock import PythonCodeBlockParser
except ImportError:
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
@ -21,7 +25,7 @@ def setup(namespace):
pytest_collect_file = Sybil(
parsers=[
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
CodeBlockParser(future_imports=['print_function']),
PythonCodeBlockParser(future_imports=['print_function']),
skip,
],
pattern='*.rst',

10
docs/contributing.rst
View File

@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
the tests with Python 3.6 use::
the tests with Python 3.7 use::
tox -e py36
tox -e py37
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
tox -e py36,py38 -p auto
tox -e py37,py38 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
tox -e py36 -- scrapy tests -n auto
tox -e py37 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:

9
docs/faq.rst
View File

@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
See :ref:`topics-request-response-ref-request-callback-arguments`.
Scrapy crashes with: ImportError: No module named win32api
----------------------------------------------------------
You need to install `pywin32`_ because of `this Twisted bug`_.
.. _pywin32: https://sourceforge.net/projects/pywin32/
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
How can I simulate a user login in my spider?
---------------------------------------------

4
docs/index.rst
View File

@ -12,6 +12,8 @@ testing.
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
.. _getting-help:
Getting help
============
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
* Search for questions on the archives of the `scrapy-users mailing list`_.
* Ask a question in the `#scrapy IRC channel`_,
* Report bugs with Scrapy in our `issue tracker`_.
* Join the Discord community `Scrapy Discord`_.
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
First steps

2
docs/intro/examples.rst
View File

@ -7,7 +7,7 @@ Examples
The best way to learn is with examples, and Scrapy is no exception. For this
reason, there is an example Scrapy project named quotesbot_, that you can use to
play and learn more about Scrapy. It contains two spiders for
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
expressions.
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.

20
docs/intro/install.rst
View File

@ -9,8 +9,8 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.6+, either the CPython implementation (default) or
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
Scrapy requires Python 3.7+, either the CPython implementation (default) or
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@ -52,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
* `twisted`_, an asynchronous networking framework
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
The minimal versions which Scrapy is tested against are:
* Twisted 14.0
* lxml 3.4
* pyOpenSSL 0.14
Scrapy may work with older versions of these packages
but it is not guaranteed it will continue working
because it’s not being tested against them.
Some of these packages themselves depends on non-Python packages
that might require additional installation steps depending on your platform.
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
@ -190,7 +180,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
successfully install Scrapy and its dependencies. Here are some proposed
solutions:
* *(Recommended)* **Don't** use system python, install a new, updated version
* *(Recommended)* **Don't** use system Python. Install a new, updated version
that doesn't conflict with the rest of your system. Here's how to do it using
the `homebrew`_ package manager:
@ -234,8 +224,8 @@ For PyPy3, only Linux installation was tested.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
On macOS, you are likely to face an issue with building the Cryptography
dependency. The solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing Scrapy). Installing on Linux has no special

10
docs/intro/overview.rst
View File

@ -4,7 +4,7 @@
Scrapy at a glance
==================
Scrapy is an application framework for crawling web sites and extracting
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
structured data which can be used for a wide range of useful applications, like
data mining, information processing or historical archival.
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
example of a Scrapy Spider using the simplest way to run a spider.
Here's the code for a spider that scrapes famous quotes from website
http://quotes.toscrape.com, following the pagination::
https://quotes.toscrape.com, following the pagination::
import scrapy
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
class QuotesSpider(scrapy.Spider):
name = 'quotes'
start_urls = [
'http://quotes.toscrape.com/tag/humor/',
'https://quotes.toscrape.com/tag/humor/',
]
def parse(self, response):
@ -45,9 +45,9 @@ http://quotes.toscrape.com, following the pagination::
Put this in a text file, name it to something like ``quotes_spider.py``
and run the spider using the :command:`runspider` command::
scrapy runspider quotes_spider.py -o quotes.jl
scrapy runspider quotes_spider.py -o quotes.jsonl
When this finishes you will have in the ``quotes.jl`` file a list of the
When this finishes you will have in the ``quotes.jsonl`` file a list of the
quotes in JSON Lines format, containing text and author, looking like this::
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}

72
docs/intro/tutorial.rst
View File

@ -7,7 +7,7 @@ Scrapy Tutorial
In this tutorial, we'll assume that Scrapy is already installed on your system.
If that's not the case, see :ref:`intro-install`.
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
that lists quotes from famous authors.
This tutorial will walk you through these tasks:
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
def start_requests(self):
urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -143,9 +143,9 @@ similar to this::
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
@ -184,8 +184,8 @@ for your spider::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
def parse(self, response):
@ -207,7 +207,7 @@ Extracting data
The best way to learn how to extract data with Scrapy is trying selectors
using the :ref:`Scrapy shell <topics-shell>`. Run::
scrapy shell 'http://quotes.toscrape.com/page/1/'
scrapy shell 'https://quotes.toscrape.com/page/1/'
.. note::
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
On Windows, use double quotes instead::
scrapy shell "http://quotes.toscrape.com/page/1/"
scrapy shell "https://quotes.toscrape.com/page/1/"
You will see something like::
[ ... Scrapy log here ... ]
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
[s] Available Scrapy objects:
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
[s] item {}
[s] request <GET http://quotes.toscrape.com/page/1/>
[s] response <200 http://quotes.toscrape.com/page/1/>
[s] request <GET https://quotes.toscrape.com/page/1/>
[s] response <200 https://quotes.toscrape.com/page/1/>
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
[s] Useful shortcuts:
@ -241,7 +241,7 @@ object:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
@ -277,9 +277,19 @@ As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
find any element matching the selection.
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
>>> response.css('noelement')[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -345,7 +355,7 @@ Extracting quotes and authors
Now that you know a bit about selection and extraction, let's complete our
spider by writing the code to extract the quotes from the web page.
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
like this:
.. code-block:: html
@ -369,7 +379,7 @@ like this:
Let's open up scrapy shell and play a bit to find out how to extract the data
we want::
$ scrapy shell 'http://quotes.toscrape.com'
$ scrapy shell 'https://quotes.toscrape.com'
We get a list of selectors for the quote HTML elements with:
@ -434,8 +444,8 @@ in the callback, as you can see below::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'http://quotes.toscrape.com/page/2/',
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
]
def parse(self, response):
@ -448,9 +458,9 @@ in the callback, as you can see below::
If you run this spider, it will output the extracted data with the log::
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
@ -472,13 +482,13 @@ to append new content to any existing file. However, appending to a JSON file
makes the file contents invalid JSON. When appending to a file, consider
using a different serialization format, such as `JSON Lines`_::
scrapy crawl quotes -o quotes.jl
scrapy crawl quotes -o quotes.jsonl
The `JSON Lines`_ format is useful because it's stream-like, you can easily
append new records to it. It doesn't have the same problem of JSON when you run
twice. Also, as each record is a separate line, you can process big files
without having to fit everything in memory, there are tools like `JQ`_ to help
doing that at the command-line.
do that at the command-line.
In small projects (like the one in this tutorial), that should be enough.
However, if you want to perform more complex things with the scraped items, you
@ -495,7 +505,7 @@ Following links
===============
Let's say, instead of just scraping the stuff from the first two pages
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
Now that you know how to extract data from pages, let's see how to follow links
from them.
@ -539,7 +549,7 @@ page, extracting data from it::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/1/',
]
def parse(self, response):
@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'http://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/1/',
]
def parse(self, response):
@ -644,7 +654,7 @@ this time for scraping author information::
class AuthorSpider(scrapy.Spider):
name = 'author'
start_urls = ['http://quotes.toscrape.com/']
start_urls = ['https://quotes.toscrape.com/']
def parse(self, response):
author_page_links = response.css('.author + a')
@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument::
name = "quotes"
def start_requests(self):
url = 'http://quotes.toscrape.com/'
url = 'https://quotes.toscrape.com/'
tag = getattr(self, 'tag', None)
if tag is not None:
url = url + 'tag/' + tag
@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument::
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
will only visit URLs from the ``humor`` tag, such as
``http://quotes.toscrape.com/tag/humor``.
``https://quotes.toscrape.com/tag/humor``.
You can :ref:`learn more about handling spider arguments here <spiderargs>`.

545
docs/news.rst
View File

@ -3,6 +3,450 @@
Release notes
=============
.. _release-2.6.1:
Scrapy 2.6.1 (2022-03-01)
-------------------------
Fixes a regression introduced in 2.6.0 that would unset the request method when
following redirects.
.. _release-2.6.0:
Scrapy 2.6.0 (2022-03-01)
-------------------------
Highlights:
* :ref:`Security fixes for cookie handling <2.6-security-fixes>`
* Python 3.10 support
* :ref:`asyncio support <using-asyncio>` is no longer considered
experimental, and works out-of-the-box on Windows regardless of your Python
version
* Feed exports now support :class:`pathlib.Path` output paths and per-feed
:ref:`item filtering <item-filter>` and
:ref:`post-processing <post-processing>`
.. _2.6-security-fixes:
Security bug fixes
~~~~~~~~~~~~~~~~~~
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
redirect response causing a new :class:`~scrapy.http.Request` object to be
scheduled, the cookies defined in the original
:class:`~scrapy.http.Request` object are no longer copied into the new
:class:`~scrapy.http.Request` object.
If you manually set the ``Cookie`` header on a
:class:`~scrapy.http.Request` object and the domain name of the redirect
URL is not an exact match for the domain of the URL of the original
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
from the new :class:`~scrapy.http.Request` object.
The old behavior could be exploited by an attacker to gain access to your
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
information.
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
.. note:: It is still possible to enable the sharing of cookies between
different domains with a shared domain suffix (e.g.
``example.com`` and any subdomain) by defining the shared domain
suffix (e.g. ``example.com``) as the cookie domain when defining
your cookies. See the documentation of the
:class:`~scrapy.http.Request` class for more information.
- When the domain of a cookie, either received in the ``Set-Cookie`` header
of a response or defined in a :class:`~scrapy.http.Request` object, is set
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
ignored unless the cookie domain is the same as the request domain.
The old behavior could be exploited by an attacker to inject cookies from a
controlled domain into your cookiejar that could be sent to other domains
not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
advisory`_ for more information.
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- The h2_ dependency is now optional, only needed to
:ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
.. _h2: https://pypi.org/project/h2/
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
for a non-POST request, now overrides the URL query string, instead of
being appended to it. (:issue:`2919`, :issue:`3579`)
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
the return value of that function, and not the ``params`` input parameter,
will determine the feed URI parameters, unless that return value is
``None``. (:issue:`4962`, :issue:`4966`)
- In :class:`scrapy.core.engine.ExecutionEngine`, methods
:meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
:meth:`~scrapy.core.engine.ExecutionEngine.download`,
:meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
now raise :exc:`RuntimeError` if called before
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
These methods used to assume that
:attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
been defined by a prior call to
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
raising :exc:`AttributeError` instead.
- If the API of the configured :ref:`scheduler <topics-scheduler>` does not
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
other exceptions would be raised at run time. (:issue:`3559`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
now been removed. (:issue:`5393`)
- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
(:issue:`5398`)
- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
(:issue:`5398`)
- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
been removed. (:issue:`4178`, :issue:`4356`)
Deprecations
~~~~~~~~~~~~
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
returning ``None`` or modifying the ``params`` input parameter is now
deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
- Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
:meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
- Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
:func:`scrapy.utils.request.request_from_dict` function.
- In :mod:`scrapy.squeues`, the following queue classes are deprecated:
:class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
:class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
:class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
instead use:
:class:`~scrapy.squeues.PickleFifoDiskQueue`,
:class:`~scrapy.squeues.PickleLifoDiskQueue`,
:class:`~scrapy.squeues.MarshalFifoDiskQueue`,
and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
a time when this class could handle multiple :class:`~scrapy.Spider`
objects at a time have been deprecated. (:issue:`5090`)
- The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
is deprecated.
- The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
:meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
- The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
instead.
- The ``spider`` parameter is deprecated for the following methods:
- :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
- :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
- :meth:`~scrapy.core.engine.ExecutionEngine.download`
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
first to set the :class:`~scrapy.Spider` object.
New features
~~~~~~~~~~~~
- You can now use :ref:`item filtering <item-filter>` to control which items
are exported to each output feed. (:issue:`4575`, :issue:`5178`,
:issue:`5161`, :issue:`5203`)
- You can now apply :ref:`post-processing <post-processing>` to feeds, and
:ref:`built-in post-processing plugins <builtin-plugins>` are provided for
output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
keys. (:issue:`5383`, :issue:`5384`)
- Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
or later will automatically switch the asyncio event loop to one that
allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
:issue:`5315`)
- The :command:`genspider` command now supports a start URL instead of a
domain name. (:issue:`4439`)
- :mod:`scrapy.utils.defer` gained 2 new functions,
:func:`~scrapy.utils.defer.deferred_to_future` and
:func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
(:issue:`5288`)
- :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
support for `temporary security credentials`_
(:setting:`AWS_SESSION_TOKEN`) and endpoint customization
(:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
(:issue:`5279`)
- :attr:`Request.cookies <scrapy.Request.cookies>` values that are
:class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
(:issue:`5252`, :issue:`5253`)
- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
the :signal:`spider_idle` signal to customize the reason why the spider is
stopping. (:issue:`5191`)
- When using
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
scheme. (:issue:`4505`, :issue:`4649`)
- All built-in queues now expose a ``peek`` method that returns the next
queue object (like ``pop``) but does not remove the returned object from
the queue. (:issue:`5112`)
If the underlying queue does not support peeking (e.g. because you are not
using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
:exc:`NotImplementedError`.
- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
an ``attributes`` attribute that makes subclassing easier. For
:class:`~scrapy.http.Request`, it also allows subclasses to work with
:func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
:issue:`5130`, :issue:`5218`)
- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
:meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
:ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
exceptions now only truncate response bodies longer than 1000 characters,
instead of those longer than 32 characters, making it easier to debug such
errors. (:issue:`4881`, :issue:`5007`)
- :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
(:issue:`5145`, :issue:`5269`)
Bug fixes
~~~~~~~~~
- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
(:issue:`4485`, :issue:`5352`)
- Removed a module-level Twisted reactor import that could prevent
:ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
- The :command:`startproject` command works with existing folders again.
(:issue:`4665`, :issue:`4676`)
- The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
(:issue:`4962`, :issue:`4966`)
- :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
- Made :func:`scrapy.utils.response.open_in_browser` support more complex
HTML. (:issue:`5319`, :issue:`5320`)
- Fixed :attr:`CSVFeedSpider.quotechar
<scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
encoding. (:issue:`5391`, :issue:`5394`)
- Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
.. _setuptools: https://pypi.org/project/setuptools/
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
now also works as expected with links that have comma-separated ``rel``
attribute values including ``nofollow``. (:issue:`5225`)
- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
<topics-feed-exports>` parameter parsing. (:issue:`5359`)
Documentation
~~~~~~~~~~~~~
- :ref:`asyncio support <using-asyncio>` is no longer considered
experimental. (:issue:`5332`)
- Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
(:issue:`4976`, :issue:`5315`)
- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
(:issue:`4484`, :issue:`4613`)
- Documented :ref:`local file naming in media pipelines
<topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
:issue:`3669`)
- Provided better context and instructions to disable the
:setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
- Documented that :ref:`reppy-parser` does not support Python 3.9+.
(:issue:`5226`, :issue:`5231`)
- Documented :ref:`the scheduler component <topics-scheduler>`.
(:issue:`3537`, :issue:`3559`)
- Documented the method used by :ref:`media pipelines
<topics-media-pipeline>` to :ref:`determine if a file has expired
<file-expiration>`. (:issue:`5120`, :issue:`5254`)
- :ref:`run-multiple-spiders` now features
:func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
- :ref:`run-multiple-spiders` now covers what happens when you define
different per-spider values for some settings that cannot differ at run
time. (:issue:`4485`, :issue:`5352`)
- Extended the documentation of the
:class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
(:issue:`5199`, :issue:`5217`)
- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
:issue:`5224`)
- Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
(:issue:`5174`, :issue:`5244`)
- Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
(:issue:`1582`)
- Added the ``body_length`` parameter to the documented signature of the
:signal:`headers_received` signal. (:issue:`5270`)
- Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
- The documentation now features the shortest import path of classes with
multiple import paths. (:issue:`2733`, :issue:`5099`)
- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
(:issue:`5395`, :issue:`5396`)
- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
- The pronunciation of the project name is now :ref:`officially
<intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
- Fixed issues and implemented minor improvements. (:issue:`3155`,
:issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
:issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
:issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
:issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
Quality Assurance
~~~~~~~~~~~~~~~~~
- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
:issue:`5265`)
- Significantly reduced memory usage by
:func:`scrapy.utils.response.response_httprepr`, used by the
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
:issue:`5374`)
- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
:issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
:issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
:issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
:issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
:issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
:issue:`5425`, :issue:`5427`)
- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
:issue:`5177`, :issue:`5200`)
- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
:issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
:issue:`5314`, :issue:`5322`)
.. _release-2.5.1:
Scrapy 2.5.1 (2021-10-05)
-------------------------
* **Security bug fix:**
If you use
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
authentication, any request exposes your credentials to the request target.
To prevent unintended exposure of authentication credentials to unintended
domains, you must now additionally set a new, additional spider attribute,
``http_auth_domain``, and point it to the specific domain to which the
authentication credentials must be sent.
If the ``http_auth_domain`` spider attribute is not set, the domain of the
first request will be considered the HTTP authentication target, and
authentication credentials will only be sent in requests targeting that
domain.
If you need to send the same HTTP authentication credentials to multiple
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
set the value of the ``Authorization`` header of your requests.
If you *really* want your spider to send the same HTTP authentication
credentials to any domain, set the ``http_auth_domain`` spider attribute
to ``None``.
Finally, if you are a user of `scrapy-splash`_, know that this version of
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
need to upgrade scrapy-splash to a greater version for it to continue to
work.
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _release-2.5.0:
Scrapy 2.5.0 (2021-04-06)
@ -940,9 +1384,8 @@ Bug fixes
* zope.interface 5.0.0 and later versions are now supported
(:issue:`4447`, :issue:`4448`)
* :meth:`Spider.make_requests_from_url
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
1.4.0, now issues a warning when used (:issue:`4412`)
* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
warning when used (:issue:`4412`)
Documentation
@ -1200,7 +1643,7 @@ New features
:issue:`4370`)
* A new ``keep_fragments`` parameter of
:func:`scrapy.utils.request.request_fingerprint` allows to generate
``scrapy.utils.request.request_fingerprint`` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
@ -1454,6 +1897,88 @@ affect subclasses:
(:issue:`3884`)
.. _release-1.8.2:
Scrapy 1.8.2 (2022-03-01)
-------------------------
**Security bug fixes:**
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
redirect response causing a new :class:`~scrapy.http.Request` object to be
scheduled, the cookies defined in the original
:class:`~scrapy.http.Request` object are no longer copied into the new
:class:`~scrapy.http.Request` object.
If you manually set the ``Cookie`` header on a
:class:`~scrapy.http.Request` object and the domain name of the redirect
URL is not an exact match for the domain of the URL of the original
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
from the new :class:`~scrapy.http.Request` object.
The old behavior could be exploited by an attacker to gain access to your
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
information.
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
.. note:: It is still possible to enable the sharing of cookies between
different domains with a shared domain suffix (e.g.
``example.com`` and any subdomain) by defining the shared domain
suffix (e.g. ``example.com``) as the cookie domain when defining
your cookies. See the documentation of the
:class:`~scrapy.http.Request` class for more information.
- When the domain of a cookie, either received in the ``Set-Cookie`` header
of a response or defined in a :class:`~scrapy.http.Request` object, is set
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
ignored unless the cookie domain is the same as the request domain.
The old behavior could be exploited by an attacker to inject cookies into
your requests to some other domains. Please, see the `mfjm-vh54-3f96
security advisory`_ for more information.
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
.. _release-1.8.1:
Scrapy 1.8.1 (2021-10-05)
-------------------------
* **Security bug fix:**
If you use
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
authentication, any request exposes your credentials to the request target.
To prevent unintended exposure of authentication credentials to unintended
domains, you must now additionally set a new, additional spider attribute,
``http_auth_domain``, and point it to the specific domain to which the
authentication credentials must be sent.
If the ``http_auth_domain`` spider attribute is not set, the domain of the
first request will be considered the HTTP authentication target, and
authentication credentials will only be sent in requests targeting that
domain.
If you need to send the same HTTP authentication credentials to multiple
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
set the value of the ``Authorization`` header of your requests.
If you *really* want your spider to send the same HTTP authentication
credentials to any domain, set the ``http_auth_domain`` spider attribute
to ``None``.
Finally, if you are a user of `scrapy-splash`_, know that this version of
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
need to upgrade scrapy-splash to a greater version for it to continue to
work.
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _release-1.8.0:
Scrapy 1.8.0 (2019-10-28)
@ -1754,7 +2279,7 @@ New features
* A new scheduler priority queue,
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
scheduling improvement on crawls targetting multiple web domains, at the
scheduling improvement on crawls targeting multiple web domains, at the
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
* A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
@ -2792,7 +3317,7 @@ Bug fixes
- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
(:issue:`2225`).
- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
Refactoring
~~~~~~~~~~~
@ -3655,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09)
- adding some xpath tips to selectors docs (:commit:`2d103e0`)
- fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`)
- get_func_args maximum recursion fix #728 (:commit:`81344ea`)
- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`)
- Updated input/output processor example according to #560. (:commit:`f7c4ea8`)
- Fixed Python syntax in tutorial. (:commit:`db59ed9`)
- Add test case for tunneling proxy (:commit:`f090260`)
- Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`)
@ -4282,7 +4807,7 @@ Code rearranged and removed
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
- Removed support for default field values in Scrapy items (:rev:`2616`)
- Removed experimental crawlspider v2 (:rev:`2632`)
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
- Removed deprecated Execution Queue (:rev:`2704`)
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
@ -4317,7 +4842,7 @@ Scrapyd changes
~~~~~~~~~~~~~~~
- Scrapyd now uses one process per spider
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default)
- A minimal web ui was added, available at http://localhost:6800 by default
- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
@ -4353,7 +4878,7 @@ New features and improvements
- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
- Support for overriding default request headers per spider (#181)
- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
- Splitted Debian package into two packages - the library and the service (#187)
- Split Debian package into two packages - the library and the service (#187)
- Scrapy log refactoring (#188)
- New extension for keeping persistent spider contexts among different runs (#203)
- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)

7
docs/topics/api.rst
View File

@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
:class:`scrapy.Spider` subclass and a
:class:`scrapy.settings.Settings` object.
.. attribute:: request_fingerprinter
The request fingerprint builder of this crawler.
This is used from extensions and middlewares to build short, unique
identifiers for requests. See :ref:`request-fingerprints`.
.. attribute:: settings
The settings manager of this crawler.

2
docs/topics/architecture.rst
View File

@ -67,7 +67,7 @@ this:
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
to crawl.
9. The process repeats (from step 1) until there are no more requests from the
9. The process repeats (from step 3) until there are no more requests from the
:ref:`Scheduler <component-scheduler>`.
Components

63
docs/topics/asyncio.rst
View File

@ -10,10 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
recommended for production environments. Future Scrapy versions
may introduce related changes without a deprecation period or
warning.
.. _install-asyncio:
@ -30,6 +26,7 @@ reactor manually. You can do that using
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
.. _using-custom-loops:
Using custom asyncio loops
@ -40,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
use it instead of the default asyncio event loop.
.. _asyncio-windows:
Windows-specific notes
======================
The Windows implementation of :mod:`asyncio` can use two event loop
implementations:
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
when using Twisted.
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
with Twisted.
So on Python 3.8+ the event loop class needs to be changed.
.. versionchanged:: 2.6.0
The event loop class is changed automatically when you change the
:setting:`TWISTED_REACTOR` setting or call
:func:`~scrapy.utils.reactor.install_reactor`.
To change the event loop class manually, call the following code before
installing the reactor::
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
You can put this in the same function that installs the reactor, if you do that
yourself, or in some code that runs before the reactor is installed, e.g.
``settings.py``.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
subprocesses (this is the case with `playwright`_), so you cannot use
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
.. _playwright: https://github.com/microsoft/playwright-python
.. _asyncio-await-dfd:
Awaiting on Deferreds
=====================
When the asyncio reactor isn't installed, you can await on Deferreds in the
coroutines directly. When it is installed, this is not possible anymore, due to
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
:class:`asyncio.Future` objects, not into
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
Futures. Scrapy provides two helpers for this:
.. autofunction:: scrapy.utils.defer.deferred_to_future
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
.. tip:: If you need to use these functions in code that aims to be compatible
with lower versions of Scrapy that do not provide these functions,
down to Scrapy 2.0 (earlier versions do not support
:mod:`asyncio`), you can copy the implementation of these functions
into your own code.

10
docs/topics/commands.rst
View File

@ -230,10 +230,16 @@ Usage example::
genspider
---------
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
* Requires project: *no*
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. versionadded:: 2.6.0
The ability to pass a URL instead of a domain.
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. note:: Even if an HTTPS URL is specified, the protocol used in
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
Usage example::

15
docs/topics/coroutines.rst
View File

@ -1,3 +1,5 @@
.. _topics-coroutines:
==========
Coroutines
==========
@ -75,23 +77,28 @@ coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
class MySpider(Spider):
class MySpiderDeferred(Spider):
# ...
async def parse_with_deferred(self, response):
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
async def parse_with_asyncio(self, response):
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
additional_data = await r.text()
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
:mod:`asyncio` loop and to use them you need to
:doc:`enable asyncio support in Scrapy<asyncio>`.
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
you need to :ref:`wrap them<asyncio-await-dfd>`.
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in callbacks,

24
docs/topics/developer-tools.rst
View File

@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
Since Developer Tools operate on a live browser DOM, what you'll actually see
when inspecting the page source is not the original HTML, but a modified one
after applying some browser clean up and executing Javascript code. Firefox,
after applying some browser clean up and executing JavaScript code. Firefox,
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ``<tbody>`` in your XPath expressions.
Therefore, you should keep in mind the following things:
* Disable Javascript while inspecting the DOM looking for XPaths to be
* Disable JavaScript while inspecting the DOM looking for XPaths to be
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
* Never use full XPath paths, use relative and clever ones based on attributes
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
copy XPaths to selected elements. Let's try it out.
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
$ scrapy shell "https://quotes.toscrape.com/"
Then, back to your web browser, right-click on the ``span`` tag, select
``Copy > XPath`` and paste it in the Scrapy shell like so:
.. invisible-code-block: python
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
type ``json``.
If we click on this request, we see that the request URL is
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
is a JSON-object that contains our quotes. We can also right-click
on the request and open ``Open in new tab`` to get a better overview.
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
page = 1
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
def parse(self, response):
data = json.loads(response.text)
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
yield {"quote": quote["text"]}
if data["has_next"]:
self.page += 1
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
yield scrapy.Request(url=url, callback=self.parse)
This spider starts at the first page of the quotes-API. With each
@ -280,7 +280,7 @@ request::
from scrapy import Request
request = Request.from_curl(
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
to identifying the correct request and replicating it in your spider.
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
.. _quotes.toscrape.com: http://quotes.toscrape.com
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
.. _quotes.toscrape.com: https://quotes.toscrape.com
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions

29
docs/topics/downloader-middleware.rst
View File

@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
methods of installed middleware is always called on every response.
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
process_request methods and reschedule the returned request. Once the newly returned
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
request is performed, the appropriate middleware chain will be called on
the downloaded response.
@ -323,8 +323,21 @@ HttpAuthMiddleware
This middleware authenticates all requests generated from certain spiders
using `Basic access authentication`_ (aka. HTTP auth).
To enable HTTP authentication from certain spiders, set the ``http_user``
and ``http_pass`` attributes of those spiders.
To enable HTTP authentication for a spider, set the ``http_user`` and
``http_pass`` spider attributes to the authentication data and the
``http_auth_domain`` spider attribute to the domain which requires this
authentication (its subdomains will be also handled in the same way).
You can set ``http_auth_domain`` to ``None`` to enable the
authentication for all requests but you risk leaking your authentication
credentials to unrelated domains.
.. warning::
In previous Scrapy versions HttpAuthMiddleware sent the authentication
data with all requests, which is a security problem if the spider
makes requests to several different domains. Currently if the
``http_auth_domain`` attribute is not set, the middleware will use the
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
@ -334,6 +347,7 @@ HttpAuthMiddleware
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
# .. rest of the spider code omitted ...
@ -352,7 +366,7 @@ HttpCacheMiddleware
This middleware provides low-level cache to all HTTP requests and responses.
It has to be combined with a cache storage backend as well as a cache policy.
Scrapy ships with three HTTP cache storage backends:
Scrapy ships with the following HTTP cache storage backends:
* :ref:`httpcache-storage-fs`
* :ref:`httpcache-storage-dbm`
@ -690,14 +704,15 @@ HttpCompressionMiddleware
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ as well as
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
installed, respectively.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.org/project/brotlipy/
.. _brotli: https://pypi.org/project/Brotli/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -1005,7 +1020,7 @@ Parsers vary in several aspects:
(shorter) rule
Performance comparison of different parsers is available at `the following link
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
<https://github.com/scrapy/scrapy/issues/3969>`_.
.. _protego-parser:

2
docs/topics/exporters.rst
View File

@ -122,7 +122,7 @@ Example::
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if field == 'price':
if name == 'price':
return f'$ {str(value)}'
return super().serialize_field(field, name, value)

20
docs/topics/feed-exports.rst
View File

@ -278,7 +278,7 @@ feed URI, allowing item delivery to start way before the end of the crawl.
Item filtering
==============
.. versionadded:: VERSION
.. versionadded:: 2.6.0
You can filter items that you want to allow for a particular feed by using the
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
@ -318,11 +318,11 @@ ItemFilter
Post-Processing
===============
.. versionadded:: VERSION
.. versionadded:: 2.6.0
Scrapy provides an option to activate plugins to post-process feeds before they are exported
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
can create your own :ref:`plugins <custom-plugins>`.
can create your own :ref:`plugins <custom-plugins>`.
These plugins can be activated through the ``postprocessing`` option of a feed.
The option must be passed a list of post-processing plugins in the order you want
@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods:
Close the target file object.
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
can then access those parameters from the ``__init__`` method of your plugin.
@ -457,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition:
If undefined or empty, all items are exported.
.. versionadded:: VERSION
.. versionadded:: 2.6.0
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
.. versionadded:: VERSION
.. versionadded:: 2.6.0
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
@ -499,7 +499,7 @@ as a fallback value if that key is not provided for a specific feed definition:
The plugins will be used in the order of the list passed.
.. versionadded:: VERSION
.. versionadded:: 2.6.0
.. setting:: FEED_EXPORT_ENCODING
@ -638,6 +638,7 @@ Default::
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@ -744,6 +745,9 @@ The function signature should be as follows:
:param spider: source spider of the feed items
:type spider: scrapy.Spider
.. caution:: The function should return a new dictionary, modifying
the received ``params`` in-place is deprecated.
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
@ -760,7 +764,7 @@ source spider in the feed URI:
#. Use ``%(spider_name)s`` in your feed URI::
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier

16
docs/topics/item-pipeline.rst
View File

@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
.. method:: from_crawler(cls, crawler)
.. classmethod:: from_crawler(cls, crawler)
If present, this classmethod is called to create a pipeline instance
If present, this class method is called to create a pipeline instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the pipeline. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for pipeline to
@ -99,11 +99,11 @@ contain a price::
raise DropItem(f"Missing price in {item}")
Write items to a JSON file
--------------------------
Write items to a JSON lines file
--------------------------------
The following pipeline stores all scraped items (from all spiders) into a
single ``items.jl`` file, containing one item per line serialized in JSON
single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
import json
@ -113,7 +113,7 @@ format::
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
self.file = open('items.jsonl', 'w')
def close_spider(self, spider):
self.file.close()
@ -190,6 +190,8 @@ item.
import scrapy
from itemadapter import ItemAdapter
from scrapy.utils.defer import maybe_deferred_to_future
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
@ -202,7 +204,7 @@ item.
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await spider.crawler.engine.download(request, spider)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
if response.status != 200:
# Error happened, return item.

5
docs/topics/items.rst
View File

@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
from dataclasses import dataclass

2
docs/topics/jobs.rst
View File

@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
.. _topics-keeping-persistent-state-between-batches:
Keeping persistent state between batches
========================================

2
docs/topics/loaders.rst
View File

@ -56,7 +56,7 @@ chapter <topics-items>`::
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
return l.load_item()

7
docs/topics/logging.rst
View File

@ -143,6 +143,7 @@ Logging settings
These settings can be used to configure the logging:
* :setting:`LOG_FILE`
* :setting:`LOG_FILE_APPEND`
* :setting:`LOG_ENABLED`
* :setting:`LOG_ENCODING`
* :setting:`LOG_LEVEL`
@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
:setting:`LOG_FILE` is set, messages sent through the root logger will be
redirected to a file named :setting:`LOG_FILE` with encoding
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
messages will be displayed on the standard error. Lastly, if
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
(discarding the output from previous runs, if any). Lastly, if
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many
HTTP 404 and 500 responses, and you want to hide all messages like this::
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
is not handled or not allowed
The first thing to note is a logger name - it is in brackets:

27
docs/topics/media-pipeline.rst
View File

@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
Additional features
===================
.. _file-expiration:
File expiration
---------------
@ -383,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
and pipeline class MyPipeline will have expiration time set to 180.
The last modified time from the file is used to determine the age of the file in days,
which is then compared to the set expiration time to determine if the file is expired.
.. _topics-images-thumbnails:
Thumbnail generation for images
@ -651,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
.. versionadded:: 2.4
The *item* parameter.
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
thumbnail download path of the image originating from the specified
:class:`response <scrapy.http.Response>`.
In addition to ``response``, this method receives the original
:class:`request <scrapy.Request>`,
``thumb_id``,
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
:class:`item <scrapy.Item>`.
You can override this method to customize the thumbnail download path of each image.
You can use the ``item`` to determine the file path based on some item
property.
By default the :meth:`thumb_path` method returns
``thumbs/<size name>/<request URL hash>.<extension>``.
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,

25
docs/topics/practices.rst
View File

@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
# Your second spider definition
...
configure_logging()
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
crawl()
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some
limitations, to use these different values. What happens in practice is
different for different settings:
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
default one) cannot be read from the per-spider settings. These are applied
when the :class:`~scrapy.crawler.CrawlerRunner` or
:class:`~scrapy.crawler.CrawlerProcess` object is created.
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
available value is used, and if a spider requests a different reactor an
exception will be raised. These are applied when the reactor is installed.
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
the first available value is used. These are applied when the reactor is
started.
.. seealso:: :ref:`run-from-script`.
.. _distributed-crawls:
@ -243,7 +262,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
cookies to spot bot behaviour
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
@ -258,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _Tor project: https://www.torproject.org/
.. _commercial support: https://scrapy.org/support/
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _scrapoxy: https://scrapoxy.io/
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

282
docs/topics/request-response.rst
View File

@ -110,6 +110,10 @@ Request objects
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
current limitation that is being worked on.
.. versionadded:: 2.6.0
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
are casted to :class:`str`.
:type cookies: dict or list
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
@ -300,7 +304,7 @@ errors if needed::
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"http://www.httphttpbinbin.org/", # DNS error expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
@ -335,6 +339,7 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
@ -360,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
main_url=failure.request.cb_kwargs['main_url'],
)
.. _request-fingerprints:
Request fingerprints
--------------------
There are some aspects of scraping, such as filtering out duplicate requests
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
unique identifier from a :class:`~scrapy.http.Request` object: a request
fingerprint.
You often do not need to worry about request fingerprints, the default request
fingerprinter works for most projects.
However, there is no universal way to generate a unique identifier from a
request, because different situations require comparing requests differently.
For example, sometimes you may need to compare URLs case-insensitively, include
URL fragments, exclude certain URL query parameters, include some or all
headers, etc.
To change how request fingerprints are built for your requests, use the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
.. setting:: REQUEST_FINGERPRINTER_CLASS
REQUEST_FINGERPRINTER_CLASS
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: VERSION
Default: :class:`scrapy.utils.request.RequestFingerprinter`
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: VERSION
Default: ``'PREVIOUS_VERSION'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'PREVIOUS_VERSION'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy PREVIOUS_VERSION and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'VERSION'``
This implementation was introduced in Scrapy VERSION to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
fingerprinting algorithm and does not log this warning (
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidade the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'PREVIOUS_VERSION'``).
.. _PREVIOUS_VERSION-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A request fingerprinter is a class that must implement the following method:
.. method:: fingerprint(self, request)
Return a :class:`bytes` object that uniquely identifies *request*.
See also :ref:`request-fingerprint-restrictions`.
:param request: request to fingerprint
:type request: scrapy.http.Request
Additionally, it may also implement the following methods:
.. classmethod:: from_crawler(cls, crawler)
If present, this class method is called to create a request fingerprinter
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
new instance of the request fingerprinter.
*crawler* provides access to all Scrapy core components like settings and
signals; it is a way for the request fingerprinter to access them and hook
its functionality into Scrapy.
:param crawler: crawler that uses this request fingerprinter
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. classmethod:: from_settings(cls, settings)
If present, and ``from_crawler`` is not defined, this class method is called
to create a request fingerprinter instance from a
:class:`~scrapy.settings.Settings` object. It must return a new instance of
the request fingerprinter.
The ``fingerprint`` method of the default request fingerprinter,
:class:`scrapy.utils.request.RequestFingerprinter`, uses
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
in your ``fingerprint`` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
For example, to take the value of a request header named ``X-ID`` into
account::
# my_project/settings.py
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
# my_project/utils.py
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
return fingerprint(request, include_headers=['X-ID'])
You can also write your own fingerprinting logic from scratch.
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
- Caching saves CPU by ensuring that fingerprints are calculated only once
per request, and not once per Scrapy component that needs the fingerprint
of a request.
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
request objects do not stay in memory forever just because you have
references to them in your cache dictionary.
For example, to take into account only the URL of a request, without any prior
URL canonicalization or taking the request method or body into account::
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.url))
self.cache[request] = fp.digest()
return self.cache[request]
If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`~scrapy.utils.request.fingerprint`. For example::
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
def fingerprint(self, request):
if 'fingerprint' in request.meta:
return request.meta['fingerprint']
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter::
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
from w3lib.url import canonicalize_url
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b'')
self.cache[request] = fp.digest()
return self.cache[request]
.. _request-fingerprint-restrictions:
Request fingerprint restrictions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Scrapy components that use request fingerprints may impose additional
restrictions on the format of the fingerprints that your :ref:`request
fingerprinter <custom-request-fingerprinter>` generates.
The following built-in Scrapy components have such restrictions:
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
value of :setting:`HTTPCACHE_STORAGE`)
Request fingerprints must be at least 1 byte long.
Path and filename length limits of the file system of
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
the following directory structure is created:
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
- first byte of a request fingerprint as hexadecimal
- fingerprint as hexadecimal
- filenames up to 16 characters long
For example, if a request fingerprint is made of 20 bytes (default),
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
and the name of your spider is ``'my_spider'`` your file system must
support a file path like::
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
The underlying DBM implementation must support keys as long as twice
the number of bytes of a request fingerprint, plus 5. For example,
if a request fingerprint is made of 20 bytes (default),
45-character-long keys must be supported.
.. _topics-request-meta:
Request.meta special keys
@ -950,6 +1222,14 @@ TextResponse objects
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
.. method:: TextResponse.urljoin(url)
Constructs an absolute url by combining the Response's base url with
a possible relative url. The base url shall be extracted from the
``<base>`` tag, or just the Response's :attr:`url` if there is no such
tag.
HtmlResponse objects
--------------------

48
docs/topics/settings.rst
View File

@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
The class used to detect and filter duplicate requests.
The default (``RFPDupeFilter``) filters based on request fingerprint using
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
override its ``request_fingerprint`` method. This method should accept
scrapy :class:`~scrapy.Request` object and return its fingerprint
(a string).
The default (``RFPDupeFilter``) filters based on the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
@ -1026,6 +1022,16 @@ Default: ``None``
File name to use for logging output. If ``None``, standard error will be used.
.. setting:: LOG_FILE_APPEND
LOG_FILE_APPEND
---------------
Default: ``True``
If ``False``, the log file specified with :setting:`LOG_FILE` will be
overwritten (discarding the output from previous runs, if any).
.. setting:: LOG_FORMAT
LOG_FORMAT
@ -1566,7 +1572,7 @@ If a reactor is already installed,
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
:exc:`Exception` if the installed reactor does not match the
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
:mod:`~twisted.internet.reactor` imports in project files and imported
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
@ -1587,7 +1593,7 @@ In order to use the reactor installed by Scrapy::
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
urls = ['https://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -1615,7 +1621,7 @@ which raises :exc:`Exception`, becomes::
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
urls = ['https://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
@ -1628,10 +1634,9 @@ which raises :exc:`Exception`, becomes::
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will not attempt to install any specific reactor, and the
default reactor defined by Twisted for the current platform will be used. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
means that Scrapy will install the default reactor defined by Twisted for the
current platform. This is to maintain backward compatibility and avoid possible
problems caused by using a non-default reactor.
For additional information, see :doc:`core/howto/choosing-reactor`.
@ -1645,8 +1650,19 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
The maximum URL length to allow for crawled URLs.
This setting can act as a stopping condition in case of URLs of ever-increasing
length, which may be caused for example by a programming error either in the
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
:setting:`DEPTH_LIMIT`.
Use ``0`` to allow URLs of any length.
The default value is copied from the `Microsoft Internet Explorer maximum URL
length`_, even though this setting exists for different reasons.
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@ -1658,7 +1674,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
The default User-Agent to use when crawling, unless overridden. This user agent is
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
there is no overridding User-Agent header specified for the request.
there is no overriding User-Agent header specified for the request.
Settings documented elsewhere:

2
docs/topics/shell.rst
View File

@ -99,7 +99,7 @@ Available Shortcuts
shortcuts
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
and update all related objects accordingly. You can optionaly ask for HTTP
and update all related objects accordingly. You can optionally ask for HTTP
3xx redirections to not be followed by passing ``redirect=False``
- ``fetch(request)`` - fetch a new response from the given request and update

24
docs/topics/signals.rst
View File

@ -51,16 +51,16 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
or :term:`awaitable objects <awaitable>` from their handlers, allowing
you to run asynchronous code that does not block Scrapy. If a signal
handler returns one of these objects, Scrapy waits for that asynchronous
operation to finish.
Let's take an example::
Let's take an example using :ref:`coroutines <topics-coroutines>`::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
start_urls = ['https://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
@ -68,17 +68,15 @@ Let's take an example::
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
async def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
return response
def parse(self, response):
for quote in response.css('div.quote'):
@ -89,7 +87,7 @@ Let's take an example::
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
.. _topics-signals-ref:
@ -413,7 +411,7 @@ headers_received
.. versionadded:: 2.5
.. signal:: headers_received
.. function:: headers_received(headers, request, spider)
.. function:: headers_received(headers, body_length, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
available for a given request, before downloading any additional content.

5
docs/topics/spider-middleware.rst
View File

@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.Request` objects and :ref:`item object
<topics-items>`.
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_spider_exception` in the following
@ -440,4 +440,3 @@ UrlLengthMiddleware
settings (see the settings documentation for more info):
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.

14
docs/topics/spiders.rst
View File

@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are
different kinds of default spiders bundled into Scrapy for different purposes.
We will talk about those types here.
.. module:: scrapy.spiders
:synopsis: Spiders base class, spider manager and spider middleware
.. _topics-spiders-ref:
scrapy.Spider
=============
.. class:: scrapy.spiders.Spider()
.. class:: scrapy.spiders.Spider
.. class:: scrapy.Spider()
This is the simplest spider, and the one from which every other spider
@ -122,6 +119,11 @@ scrapy.Spider
send log messages through it as described on
:ref:`topics-logging-from-spiders`.
.. attribute:: state
A dict you can use to persist some spider state between batches.
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
.. method:: from_crawler(crawler, *args, **kwargs)
This is the class method used by Scrapy to create your spiders.
@ -367,7 +369,7 @@ CrawlSpider
described below. If multiple rules match the same link, the first one
will be used, according to the order they're defined in this attribute.
This spider also exposes an overrideable method:
This spider also exposes an overridable method:
.. method:: parse_start_url(response, **kwargs)
@ -529,7 +531,7 @@ XMLFeedSpider
itertag = 'n:url'
# ...
Apart from these new attributes, this spider has the following overrideable
Apart from these new attributes, this spider has the following overridable
methods too:
.. method:: adapt_response(response)

2
docs/versioning.rst
View File

@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
large changes.
* *B* is the release number. This will include many changes including features
and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
keep these cases at a minimum.
* *C* is the bugfix release number.
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,

2
extras/qpsclient.py
View File

@ -1,5 +1,5 @@
"""
A spider that generate light requests to meassure QPS troughput
A spider that generate light requests to meassure QPS throughput
usage:

1
pylintrc
View File

@ -112,6 +112,7 @@ disable=abstract-method,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-implicit-booleaness-not-comparison,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

3
pytest.ini
View File

@ -20,5 +20,4 @@ addopts =
--ignore=docs/utils
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
filterwarnings=
ignore::DeprecationWarning:twisted.web.test.test_webclient
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed

2
scrapy/VERSION
View File

@ -1 +1 @@
2.5.0
2.6.1

4
scrapy/__init__.py
View File

@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 6):
print("Scrapy %s requires Python 3.6+" % __version__)
if sys.version_info < (3, 7):
print(f"Scrapy {__version__} requires Python 3.7+")
sys.exit(1)

14
scrapy/cmdline.py
View File

@ -1,13 +1,13 @@
import sys
import os
import optparse
import argparse
import cProfile
import inspect
import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:])
opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings)

79
scrapy/commands/__init__.py
View File

@ -2,7 +2,7 @@
Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
import argparse
from typing import Any, Dict
from twisted.python import failure
@ -43,14 +43,14 @@ class ScrapyCommand:
def long_desc(self):
"""A long description of the command. Return short description when not
available. It cannot contain newlines, since contents will be formatted
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
"""
return self.short_desc()
def help(self):
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines, since no post-formatting will
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
@ -59,22 +59,20 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
return super()._join_parts(parts)
def format_part_strings(self, part_strings):
"""
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

8
scrapy/commands/check.py
View File

@ -49,10 +49,10 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
def run(self, args, opts):
# load contracts

10
scrapy/commands/fetch.py
View File

@ -26,11 +26,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

20
scrapy/commands/genspider.py
View File

@ -44,16 +44,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:

47
scrapy/commands/parse.py
View File

@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
def max_level(self):
@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
if self.spidercls:
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):
self.crawler_process.crawl(self.spidercls, **opts.spargs)

20
scrapy/commands/settings.py
View File

@ -18,16 +18,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

14
scrapy/commands/shell.py
View File

@ -33,12 +33,12 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
def _start_crawler_thread(self):
t = Thread(target=self.crawler_process.start,
kwargs={'stop_after_crawl': False})
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
t.daemon = True
t.start()

4
scrapy/commands/version.py
View File

@ -16,8 +16,8 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:

3
scrapy/commands/view.py
View File

@ -1,3 +1,4 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
@ -12,7 +13,7 @@ class Command(fetch.Command):
def add_options(self, parser):
super().add_options(parser)
parser.remove_option("--headers")
parser.add_argument('--headers', help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

12
scrapy/core/downloader/contextfactory.py
View File

@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
msg = (
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
"a `method` argument (type OpenSSL.SSL method, e.g. "
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
"context factory class to handle them or ignore them."
)
warnings.warn(msg)
return context_factory

6
scrapy/core/downloader/handlers/ftp.py
View File

@ -102,11 +102,11 @@ class FTPDownloadHandler:
def _build_response(self, result, request, protocol):
self.result = result
respcls = responsetypes.from_args(url=request.url)
protocol.close()
body = protocol.filename or protocol.body.read()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
body = to_bytes(protocol.filename or protocol.body.read())
respcls = responsetypes.from_args(url=request.url, body=body)
return respcls(url=request.url, status=200, body=body, headers=headers)
def _failed(self, result, request):
message = result.getErrorMessage()

2
scrapy/core/downloader/handlers/http11.py
View File

@ -213,7 +213,7 @@ class TunnelingAgent(Agent):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
key += self._proxyConf
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,

5
scrapy/core/downloader/middleware.py
View File

@ -3,7 +3,7 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
from typing import Callable, Union
from typing import Callable, Union, cast
from twisted.internet import defer
from twisted.python.failure import Failure
@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
@defer.inlineCallbacks
def process_request(request: Request):
for method in self.methods['process_request']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
for method in self.methods['process_response']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def process_exception(failure: Failure):
exception = failure.value
for method in self.methods['process_exception']:
method = cast(Callable, method)
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(

8
scrapy/core/downloader/tls.py
View File

@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
verifyHostname(connection, self._hostnameASCII)
except (CertificateError, VerificationError) as e:
logger.warning(
'Remote certificate is not valid for hostname "{}"; {}'.format(
self._hostnameASCII, e))
'Remote certificate is not valid for hostname "%s"; %s',
self._hostnameASCII, e)
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
'from host "{}" (exception: {})'.format(
self._hostnameASCII, repr(e)))
'from host "%s" (exception: %r)',
self._hostnameASCII, e)
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

5
scrapy/core/downloader/webclient.py
View File

@ -3,7 +3,7 @@ from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from twisted.internet import defer, reactor
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
def _set_connection_attributes(self, request):
@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
p.followRedirect = self.followRedirect
p.afterFoundGet = self.afterFoundGet
if self.timeout:
from twisted.internet import reactor
timeoutCall = reactor.callLater(self.timeout, p.timeout)
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
return p

4
scrapy/core/http2/stream.py
View File

@ -285,8 +285,8 @@ class Stream:
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
bytes_to_send_size = bytes_to_send_size - chunk_size
self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
bytes_to_send_size -= chunk_size
self.metadata['remaining_content_length'] -= chunk_size
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])

2
scrapy/core/scraper.py
View File

@ -156,7 +156,7 @@ class Scraper:
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallback(callback, **result.request.cb_kwargs)
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)

3
scrapy/core/spidermw.py
View File

@ -4,7 +4,7 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
from itertools import islice
from typing import Any, Callable, Generator, Iterable, Union
from typing import Any, Callable, Generator, Iterable, Union, cast
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
spider: Spider) -> Any:
for method in self.methods['process_spider_input']:
method = cast(Callable, method)
try:
result = method(response=response, spider=spider)
if result is not None:

47
scrapy/crawler.py
View File

@ -25,6 +25,7 @@ from scrapy.utils.log import (
configure_logging,
get_scrapy_root_handler,
install_scrapy_root_handler,
log_reactor_info,
log_scrapy_info,
LogCounterHandler,
)
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
class Crawler:
def __init__(self, spidercls, settings=None):
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
if isinstance(spidercls, Spider):
raise ValueError('The spidercls argument must be a class, not an object')
@ -50,6 +51,7 @@ class Crawler:
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@ -69,6 +71,26 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,
)
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
else:
from twisted.internet import default
default.install()
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
@ -153,7 +175,6 @@ class CrawlerRunner:
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
self._handle_twisted_reactor()
@property
def spiders(self):
@ -247,10 +268,6 @@ class CrawlerRunner:
while self._active:
yield defer.DeferredList(self._active)
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
class CrawlerProcess(CrawlerRunner):
"""
@ -278,7 +295,6 @@ class CrawlerProcess(CrawlerRunner):
def __init__(self, settings=None, install_root_handler=True):
super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
@ -298,7 +314,12 @@ class CrawlerProcess(CrawlerRunner):
{'signame': signame})
reactor.callFromThread(self._stop_reactor)
def start(self, stop_after_crawl=True):
def _create_crawler(self, spidercls):
if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings, init_reactor=True)
def start(self, stop_after_crawl=True, install_signal_handlers=True):
"""
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@ -309,6 +330,9 @@ class CrawlerProcess(CrawlerRunner):
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished
:param bool install_signal_handlers: whether to install the shutdown
handlers (default: True)
"""
from twisted.internet import reactor
if stop_after_crawl:
@ -318,6 +342,8 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
if install_signal_handlers:
install_shutdown_handlers(self._signal_shutdown)
resolver_class = load_object(self.settings["DNS_RESOLVER"])
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
resolver.install_on_reactor()
@ -337,8 +363,3 @@ class CrawlerProcess(CrawlerRunner):
reactor.stop()
except RuntimeError: # raised if already stopped or in shutdown stage
pass
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
super()._handle_twisted_reactor()

38
scrapy/downloadermiddlewares/cookies.py
View File

@ -1,15 +1,26 @@
import logging
from collections import defaultdict
from tldextract import TLDExtract
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
_split_domain = TLDExtract(include_psl_private_domains=True)
def _is_public_domain(domain):
parts = _split_domain(domain)
return not parts.domain
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
@ -23,14 +34,29 @@ class CookiesMiddleware:
raise NotConfigured
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
def _process_cookies(self, cookies, *, jar, request):
for cookie in cookies:
cookie_domain = cookie.domain
if cookie_domain.startswith('.'):
cookie_domain = cookie_domain[1:]
request_domain = urlparse_cached(request).hostname.lower()
if cookie_domain and _is_public_domain(cookie_domain):
if cookie_domain != request_domain:
continue
cookie.domain = request_domain
jar.set_cookie_if_ok(cookie, request)
def process_request(self, request, spider):
if request.meta.get('dont_merge_cookies', False):
return
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
for cookie in self._get_request_cookies(jar, request):
jar.set_cookie_if_ok(cookie, request)
cookies = self._get_request_cookies(jar, request)
self._process_cookies(cookies, jar=jar, request=request)
# set Cookie header
request.headers.pop('Cookie', None)
@ -44,7 +70,9 @@ class CookiesMiddleware:
# extract cookies from Set-Cookie and drop invalid/expired cookies
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
jar.extract_cookies(response, request)
cookies = jar.make_cookies(response, request)
self._process_cookies(cookies, jar=jar, request=request)
self._debug_set_cookie(response, spider)
return response
@ -80,8 +108,8 @@ class CookiesMiddleware:
logger.warning(msg.format(request, cookie, key))
return
continue
if isinstance(cookie[key], str):
decoded[key] = cookie[key]
if isinstance(cookie[key], (bool, float, int, str)):
decoded[key] = str(cookie[key])
else:
try:
decoded[key] = cookie[key].decode("utf8")

21
scrapy/downloadermiddlewares/httpauth.py
View File

@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
See documentation in docs/topics/downloader-middleware.rst
"""
import warnings
from w3lib.http import basic_auth_header
from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import url_is_from_any_domain
class HttpAuthMiddleware:
@ -24,8 +28,23 @@ class HttpAuthMiddleware:
pwd = getattr(spider, 'http_pass', '')
if usr or pwd:
self.auth = basic_auth_header(usr, pwd)
if not hasattr(spider, 'http_auth_domain'):
warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
'problems if the spider makes requests to several different domains. http_auth_domain '
'will be set to the domain of the first request, please set it to the correct value '
'explicitly.',
category=ScrapyDeprecationWarning)
self.domain_unset = True
else:
self.domain = spider.http_auth_domain
self.domain_unset = False
def process_request(self, request, spider):
auth = getattr(self, 'auth', None)
if auth and b'Authorization' not in request.headers:
request.headers[b'Authorization'] = auth
domain = urlparse_cached(request).hostname
if self.domain_unset:
self.domain = domain
self.domain_unset = False
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
request.headers[b'Authorization'] = auth

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -70,7 +70,7 @@ class HttpCacheMiddleware:
self.stats.inc_value('httpcache/miss', spider=spider)
if self.ignore_missing:
self.stats.inc_value('httpcache/ignore', spider=spider)
raise IgnoreRequest("Ignored request not in cache: %s" % request)
raise IgnoreRequest(f"Ignored request not in cache: {request}")
return None # first time request
# Return cached response only if not expired

30
scrapy/downloadermiddlewares/redirect.py
View File

@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
from scrapy.http import HtmlResponse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
def _build_redirect_request(source_request, *, url, **kwargs):
redirect_request = source_request.replace(
url=url,
**kwargs,
cookies=None,
)
if 'Cookie' in redirect_request.headers:
source_request_netloc = urlparse_cached(source_request).netloc
redirect_request_netloc = urlparse_cached(redirect_request).netloc
if source_request_netloc != redirect_request_netloc:
del redirect_request.headers['Cookie']
return redirect_request
class BaseRedirectMiddleware:
enabled_setting = 'REDIRECT_ENABLED'
@ -47,10 +62,15 @@ class BaseRedirectMiddleware:
raise IgnoreRequest("max redirections reached")
def _redirect_request_using_get(self, request, redirect_url):
redirected = request.replace(url=redirect_url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
return redirected
redirect_request = _build_redirect_request(
request,
url=redirect_url,
method='GET',
body='',
)
redirect_request.headers.pop('Content-Type', None)
redirect_request.headers.pop('Content-Length', None)
return redirect_request
class RedirectMiddleware(BaseRedirectMiddleware):
@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
redirected_url = urljoin(request.url, location)
if response.status in (301, 307, 308) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
redirected = _build_redirect_request(request, url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
redirected = self._redirect_request_using_get(request, redirected_url)

2
scrapy/downloadermiddlewares/retry.py
View File

@ -2,7 +2,7 @@
An extension to retry failed requests that are potentially caused by temporary
problems such as a connection timeout or HTTP 500 error.
You can change the behaviour of this middleware by modifing the scraping settings:
You can change the behaviour of this middleware by modifying the scraping settings:
RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry

22
scrapy/downloadermiddlewares/stats.py
View File

@ -1,7 +1,22 @@
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import global_object_name, to_bytes
from scrapy.utils.request import request_httprepr
from scrapy.utils.response import response_httprepr
from scrapy.utils.python import global_object_name
from twisted.web import http
def get_header_size(headers):
size = 0
for key, value in headers.items():
if isinstance(value, (list, tuple)):
for v in value:
size += len(b": ") + len(key) + len(v)
return size + len(b'\r\n') * (len(headers.keys()) - 1)
def get_status_size(response_status):
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
class DownloaderStats:
@ -24,7 +39,8 @@ class DownloaderStats:
def process_response(self, request, response, spider):
self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
reslen = len(response_httprepr(response))
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response

49
scrapy/dupefilters.py
View File

@ -1,14 +1,16 @@
import logging
import os
from typing import Optional, Set, Type, TypeVar
from warnings import warn
from twisted.internet.defer import Deferred
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, request_fingerprint
from scrapy.utils.request import referer_str, RequestFingerprinter
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
class RFPDupeFilter(BaseDupeFilter):
"""Request Fingerprint duplicates filter"""
def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
def __init__(
self,
path: Optional[str] = None,
debug: bool = False,
*,
fingerprinter=None,
) -> None:
self.file = None
self.fingerprinter = fingerprinter or RequestFingerprinter()
self.fingerprints: Set[str] = set()
self.logdupes = True
self.debug = debug
@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
debug = settings.getbool('DUPEFILTER_DEBUG')
return cls(job_dir(settings), debug)
try:
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
except TypeError:
warn(
"RFPDupeFilter subclasses must either modify their '__init__' "
"method to support a 'fingerprinter' parameter or reimplement "
"the 'from_settings' class method.",
ScrapyDeprecationWarning,
)
result = cls(job_dir(settings), debug)
result.fingerprinter = fingerprinter
return result
@classmethod
def from_crawler(cls, crawler):
try:
return cls.from_settings(
crawler.settings,
fingerprinter=crawler.request_fingerprinter,
)
except TypeError:
warn(
"RFPDupeFilter subclasses must either modify their overridden "
"'__init__' method and 'from_settings' class method to "
"support a 'fingerprinter' parameter, or reimplement the "
"'from_crawler' class method.",
ScrapyDeprecationWarning,
)
result = cls.from_settings(crawler.settings)
result.fingerprinter = crawler.request_fingerprinter
return result
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
return False
def request_fingerprint(self, request: Request) -> str:
return request_fingerprint(request)
return self.fingerprinter.fingerprint(request).hex()
def close(self, reason: str) -> None:
if self.file:

6
scrapy/exporters.py
View File

@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import _BaseItem
from scrapy.item import Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
@ -30,7 +30,7 @@ class BaseItemExporter:
self._configure(kwargs, dont_fail=dont_fail)
def _configure(self, options, dont_fail=False):
"""Configure the exporter by poping options from the ``options`` dict.
"""Configure the exporter by popping options from the ``options`` dict.
If dont_fail is set, it won't raise an exception on unexpected options
(useful for using with keyword arguments in subclasses ``__init__`` methods)
"""
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
return serializer(value)
def _serialize_value(self, value):
if isinstance(value, _BaseItem):
if isinstance(value, Item):
return self.export_item(value)
elif is_item(value):
return dict(self._serialize_item(value))

59
scrapy/extensions/feedexport.py
View File

@ -11,14 +11,14 @@ import sys
import warnings
from datetime import datetime
from tempfile import NamedTemporaryFile
from typing import Any, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from zope.interface import implementer, Interface
from scrapy import signals
from scrapy import signals, Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.boto import is_botocore_available
@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
kwargs['feed_options'] = feed_options
else:
warnings.warn(
"{} does not support the 'feed_options' keyword argument. Add a "
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove this "
"warning. This parameter will become mandatory in a future "
"version of Scrapy."
.format(builder.__qualname__),
"version of Scrapy.",
category=ScrapyDeprecationWarning
)
return builder(*preargs, uri, *args, **kwargs)
@ -356,32 +355,28 @@ class FeedExporter:
# properly closed.
return defer.maybeDeferred(slot.storage.store, slot.file)
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
)
d.addErrback(
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
self._handle_store_error, logmsg, spider, type(slot.storage).__name__
)
return d
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
def _handle_store_error(self, f, logmsg, spider, slot_type):
logger.error(
logfmt % "Error storing", largs,
"Error storing %s", logmsg,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
def _handle_store_success(self, f, logmsg, spider, slot_type):
logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
"Stored %s", logmsg,
extra={'spider': spider}
)
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
@ -474,10 +469,10 @@ class FeedExporter:
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
logger.error(
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
'%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
''.format(uri_template)
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
uri_template
)
return False
return True
@ -526,10 +521,15 @@ class FeedExporter:
instance = build_instance(feedcls)
method_name = '__new__'
if instance is None:
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
return instance
def _get_uri_params(self, spider, uri_params, slot=None):
def _get_uri_params(
self,
spider: Spider,
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
slot: Optional[_FeedSlot] = None,
) -> dict:
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
@ -537,9 +537,18 @@ class FeedExporter:
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
params['batch_time'] = utc_now.isoformat().replace(':', '-')
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
uripar_function(params, spider)
return params
original_params = params.copy()
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
new_params = uripar_function(params, spider)
if new_params is None or original_params != params:
warnings.warn(
'Modifying the params dictionary in-place in the function defined in '
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
'setting is deprecated. The function must return a new dictionary '
'instead.',
category=ScrapyDeprecationWarning
)
return new_params if new_params is not None else params
def _load_filter(self, feed_options):
# load the item filter if declared else load the default filter class

22
scrapy/extensions/httpcache.py
View File

@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@ -226,7 +225,9 @@ class DbmCacheStorage:
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):
self.db.close()
@ -239,12 +240,12 @@ class DbmCacheStorage:
status = data['status']
headers = Headers(data['headers'])
body = data['body']
respcls = responsetypes.from_args(headers=headers, url=url)
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
key = self._request_key(request)
key = self._fingerprinter.fingerprint(request).hex()
data = {
'status': response.status,
'url': response.url,
@ -255,7 +256,7 @@ class DbmCacheStorage:
self.db[f'{key}_time'] = str(time())
def _read_data(self, spider, request):
key = self._request_key(request)
key = self._fingerprinter.fingerprint(request).hex()
db = self.db
tkey = f'{key}_time'
if tkey not in db:
@ -267,9 +268,6 @@ class DbmCacheStorage:
return pickle.loads(db[f'{key}_data'])
def _request_key(self, request):
return request_fingerprint(request)
class FilesystemCacheStorage:
@ -280,9 +278,11 @@ class FilesystemCacheStorage:
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
self._fingerprinter = spider.crawler.request_fingerprinter
def close_spider(self, spider):
pass
@ -299,7 +299,7 @@ class FilesystemCacheStorage:
url = metadata.get('response_url')
status = metadata['status']
headers = Headers(headers_raw_to_dict(rawheaders))
respcls = responsetypes.from_args(headers=headers, url=url)
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
@ -329,7 +329,7 @@ class FilesystemCacheStorage:
f.write(request.body)
def _get_request_path(self, spider, request):
key = request_fingerprint(request)
key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):

10
scrapy/extensions/memusage.py
View File

@ -33,8 +33,8 @@ class MemoryUsage:
self.crawler = crawler
self.warned = False
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
self.mail = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@ -77,7 +77,7 @@ class MemoryUsage:
def _check_limit(self):
if self.get_virtual_size() > self.limit:
self.crawler.stats.set_value('memusage/limit_reached', 1)
mem = self.limit/1024/1024
mem = self.limit / 1024 / 1024
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:
@ -94,11 +94,11 @@ class MemoryUsage:
self.crawler.stop()
def _check_warning(self):
if self.warned: # warn only once
if self.warned: # warn only once
return
if self.get_virtual_size() > self.warning:
self.crawler.stats.set_value('memusage/warning_reached', 1)
mem = self.warning/1024/1024
mem = self.warning / 1024 / 1024
logger.warning("Memory usage reached %(memusage)dM",
{'memusage': mem}, extra={'crawler': self.crawler})
if self.notify_mails:

1
scrapy/extensions/statsmailer.py
View File

@ -8,6 +8,7 @@ from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
class StatsMailer:
def __init__(self, stats, recipients, mail):

6
scrapy/http/cookies.py
View File

@ -142,10 +142,6 @@ class WrappedRequest:
"""
return self.request.meta.get('is_unverifiable', False)
def get_origin_req_host(self):
return urlparse_cached(self.request).hostname
# python3 uses attributes instead of methods
@property
def full_url(self):
return self.get_full_url()
@ -164,7 +160,7 @@ class WrappedRequest:
@property
def origin_req_host(self):
return self.get_origin_req_host()
return urlparse_cached(self.request).hostname
def has_header(self, name):
return name in self.request.headers

9
scrapy/http/response/text.py
View File

@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
"""
import json
import warnings
from contextlib import suppress
from typing import Generator, Tuple
from urllib.parse import urljoin
@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs, to_unicode
@ -66,13 +64,6 @@ class TextResponse(Response):
or self._body_declared_encoding()
)
def body_as_unicode(self):
"""Return body as unicode"""
warnings.warn('Response.body_as_unicode() is deprecated, '
'please use Response.text instead.',
ScrapyDeprecationWarning, stacklevel=2)
return self.text
def json(self):
"""
.. versionadded:: 2.2

87
scrapy/item.py
View File

@ -9,45 +9,15 @@ from collections.abc import MutableMapping
from copy import deepcopy
from pprint import pformat
from typing import Dict
from warnings import warn
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
class _BaseItem(object_ref):
"""
Temporary class used internally to avoid the deprecation
warning raised by isinstance checks using BaseItem.
"""
pass
class _BaseItemMeta(ABCMeta):
def __instancecheck__(cls, instance):
if cls is BaseItem:
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__instancecheck__(instance)
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
"""
Deprecated, please use :class:`scrapy.item.Item` instead
"""
def __new__(cls, *args, **kwargs):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__new__(cls, *args, **kwargs)
class Field(dict):
"""Container of field metadata"""
class ItemMeta(_BaseItemMeta):
class ItemMeta(ABCMeta):
"""Metaclass_ of :class:`Item` that handles field definitions.
.. _metaclass: https://realpython.com/python-metaclasses
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
return super().__new__(mcs, class_name, bases, new_attrs)
class DictItem(MutableMapping, BaseItem):
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
"""
Base class for scraped items.
fields: Dict[str, Field] = {}
In Scrapy, an object is considered an ``item`` if it is an instance of either
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
spider callback is evaluated, only instances of :class:`Item` or
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
def __new__(cls, *args, **kwargs):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super().__new__(cls, *args, **kwargs)
If you need instances of a custom class to be considered items by Scrapy,
you must inherit from either :class:`Item` or :class:`dict`.
Items must declare :class:`Field` attributes, which are processed and stored
in the ``fields`` attribute. This restricts the set of allowed field names
and prevents typos, raising ``KeyError`` when referring to undefined fields.
Additionally, fields can be used to define metadata and control the way
data is processed internally. Please refer to the :ref:`documentation
about fields <topics-items-fields>` for additional information.
Unlike instances of :class:`dict`, instances of :class:`Item` may be
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
"""
fields: Dict[str, Field]
def __init__(self, *args, **kwargs):
self._values = {}
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
def __iter__(self):
return iter(self._values)
__hash__ = BaseItem.__hash__
__hash__ = object_ref.__hash__
def keys(self):
return self._values.keys()
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
"""Return a :func:`~copy.deepcopy` of this item.
"""
return deepcopy(self)
class Item(DictItem, metaclass=ItemMeta):
"""
Base class for scraped items.
In Scrapy, an object is considered an ``item`` if it is an instance of either
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
spider callback is evaluated, only instances of :class:`Item` or
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
If you need instances of a custom class to be considered items by Scrapy,
you must inherit from either :class:`Item` or :class:`dict`.
Items must declare :class:`Field` attributes, which are processed and stored
in the ``fields`` attribute. This restricts the set of allowed field names
and prevents typos, raising ``KeyError`` when referring to undefined fields.
Additionally, fields can be used to define metadata and control the way
data is processed internally. Please refer to the :ref:`documentation
about fields <topics-items-fields>` for additional information.
Unlike instances of :class:`dict`, instances of :class:`Item` may be
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
"""

2
scrapy/linkextractors/lxmlhtml.py
View File

@ -88,7 +88,7 @@ class LxmlParserLinkExtractor:
def _process_links(self, links):
""" Normalize and filter extracted links
The subclass should override it if neccessary
The subclass should override it if necessary
"""
return self._deduplicate_if_needed(links)

5
scrapy/loader/__init__.py
View File

@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader):
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
if selector is None and response is not None:
selector = self.default_selector_class(response)
try:
selector = self.default_selector_class(response)
except AttributeError:
selector = None
context.update(response=response)
super().__init__(item=item, selector=selector, parent=parent, **context)

17
scrapy/middleware.py
View File

@ -1,7 +1,7 @@
import logging
import pprint
from collections import defaultdict, deque
from typing import Callable, Deque, Dict
from typing import Callable, Deque, Dict, Optional, cast, Iterable
from twisted.internet.defer import Deferred
@ -9,7 +9,7 @@ from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.settings import Settings
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
from scrapy.utils.defer import process_parallel, process_chain
logger = logging.getLogger(__name__)
@ -21,7 +21,8 @@ class MiddlewareManager:
def __init__(self, *middlewares):
self.middlewares = middlewares
self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
# Optional because process_spider_output and process_spider_exception can be None
self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@ -64,14 +65,12 @@ class MiddlewareManager:
self.methods['close_spider'].appendleft(mw.close_spider)
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
return process_parallel(self.methods[methodname], obj, *args)
methods = cast(Iterable[Callable], self.methods[methodname])
return process_parallel(methods, obj, *args)
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
return process_chain_both(self.methods[cb_methodname],
self.methods[eb_methodname], obj, *args)
methods = cast(Iterable[Callable], self.methods[methodname])
return process_chain(methods, obj, *args)
def open_spider(self, spider: Spider) -> Deferred:
return self._process_parallel('open_spider', spider)

18
scrapy/pipelines/files.py
View File

@ -85,7 +85,7 @@ class S3FilesStore:
AWS_USE_SSL = None
AWS_VERIFY = None
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
HEADERS = {
'Cache-Control': 'max-age=172800',
}
@ -142,7 +142,7 @@ class S3FilesStore:
**extra)
def _headers_to_botocore_kwargs(self, headers):
""" Convert headers to botocore keyword agruments.
""" Convert headers to botocore keyword arguments.
"""
# This is required while we need to support both boto and botocore.
mapping = CaselessDict({
@ -190,7 +190,7 @@ class GCSFilesStore:
CACHE_CONTROL = 'max-age=172800'
# The bucket's default object ACL will be applied to the object.
# Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
POLICY = None
def __init__(self, uri):
@ -222,8 +222,8 @@ class GCSFilesStore:
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
blob_path = self._get_blob_path(path)
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
@ -231,8 +231,12 @@ class GCSFilesStore:
else:
return 'application/octet-stream'
def _get_blob_path(self, path):
return self.prefix + path
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob_path = self._get_blob_path(path)
blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
@ -291,7 +295,7 @@ class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
This pipeline tries to minimize network transfers and file processing,
doing stat of the files and determining if file is new, uptodate or
doing stat of the files and determining if file is new, up-to-date or
expired.
``new`` files are those that pipeline never processed and needs to be

4
scrapy/pipelines/images.py
View File

@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
yield path, image, buf
for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'full/{image_guid}.jpg'
def thumb_path(self, request, thumb_id, response=None, info=None):
def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'

6
scrapy/pipelines/media.py
View File

@ -11,7 +11,6 @@ from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.log import failure_to_exc_info
@ -77,6 +76,7 @@ class MediaPipeline:
except AttributeError:
pipe = cls()
pipe.crawler = crawler
pipe._fingerprinter = crawler.request_fingerprinter
return pipe
def open_spider(self, spider):
@ -90,7 +90,7 @@ class MediaPipeline:
return dfd.addCallback(self.item_completed, item, info)
def _process_request(self, request, info, item):
fp = request_fingerprint(request)
fp = self._fingerprinter.fingerprint(request)
cb = request.callback or (lambda _: _)
eb = request.errback
request.callback = None
@ -121,7 +121,7 @@ class MediaPipeline:
def _make_compatible(self):
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
methods = [
"file_path", "media_to_download", "media_downloaded",
"file_path", "thumb_path", "media_to_download", "media_downloaded",
"file_downloaded", "image_downloaded", "get_images"
]

6
scrapy/settings/__init__.py
View File

@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
return len(self.attributes)
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in self.items()}
def _get_key(self, key_value):
return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
else str(key_value))
def copy_to_dict(self):
"""
Make a copy of current settings and convert to a dict.

5
scrapy/settings/default_settings.py
View File

@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
@ -207,6 +208,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
LOG_STDOUT = False
LOG_LEVEL = 'DEBUG'
LOG_FILE = None
LOG_FILE_APPEND = True
LOG_SHORT_NAMES = False
SCHEDULER_DEBUG = False
@ -245,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]

6
scrapy/spiders/feed.py
View File

@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):
return response
def parse_node(self, response, selector):
"""This method must be overriden with your custom spider functionality"""
"""This method must be overridden with your custom spider functionality"""
if hasattr(self, 'parse_item'): # backward compatibility
return self.parse_item(response, selector)
raise NotImplementedError
@ -113,7 +113,7 @@ class CSVFeedSpider(Spider):
return response
def parse_row(self, response, row):
"""This method must be overriden with your custom spider functionality"""
"""This method must be overridden with your custom spider functionality"""
raise NotImplementedError
def parse_rows(self, response):
@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
process_results methods for pre and post-processing purposes.
"""
for row in csviter(response, self.delimiter, self.headers, self.quotechar):
for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
ret = iterate_spider_output(self.parse_row(response, row))
for result_item in self.process_results(response, ret):
yield result_item

3
scrapy/templates/project/module/settings.py.tmpl
View File

@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'

2
scrapy/utils/conf.py
View File

@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
message = (
'The -t command line option is deprecated in favor of '
'specifying the output format within the output URI. See the '
'documentation of the -o and -O options for more information.',
'documentation of the -o and -O options for more information.'
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
return {output[0]: {'format': output_format}}

2
scrapy/utils/console.py
View File

@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
@wraps(_embed_ipython_shell)
def wrapper(namespace=namespace, banner=''):
config = load_default_config()
# Always use .instace() to ensure _instance propagation to all parents
# Always use .instance() to ensure _instance propagation to all parents
# this is needed for <TAB> completion works well for new imports
# and clear the instance to always have the fresh env
# on repeated breaks like with inspect_response()

2
scrapy/utils/datatypes.py
View File

@ -41,7 +41,7 @@ class CaselessDict(dict):
return key.lower()
def normvalue(self, value):
"""Method to normalize values prior to be setted"""
"""Method to normalize values prior to be set"""
return value
def get(self, key, def_val=None):

65
scrapy/utils/defer.py
View File

@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds
"""
import asyncio
import inspect
from collections.abc import Coroutine
from asyncio import Future
from functools import wraps
from typing import Any, Callable, Generator, Iterable
from typing import (
Any,
Callable,
Coroutine,
Generator,
Iterable,
Union
)
from twisted.internet import defer
from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
@ -34,7 +41,7 @@ def defer_succeed(result) -> Deferred:
"""Same as twisted.internet.defer.succeed but delay calling callback until
next reactor loop
It delays by 100ms so reactor has a chance to go trough readers and writers
It delays by 100ms so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
from twisted.internet import reactor
@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
return defer.fail(result)
else:
return defer.succeed(result)
def deferred_to_future(d: Deferred) -> Future:
"""
.. versionadded:: 2.6.0
Return an :class:`asyncio.Future` object that wraps *d*.
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
callables defined as coroutines <coroutine-support>`, you can only await on
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
allows you to wait on them::
class MySpider(Spider):
...
async def parse(self, response):
d = treq.get('https://example.com/additional')
additional_response = await deferred_to_future(d)
"""
return d.asFuture(asyncio.get_event_loop())
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
"""
.. versionadded:: 2.6.0
Return *d* as an object that can be awaited from a :ref:`Scrapy callable
defined as a coroutine <coroutine-support>`.
What you can await in Scrapy callables defined as coroutines depends on the
value of :setting:`TWISTED_REACTOR`:
- When not using the asyncio reactor, you can only await on
:class:`~twisted.internet.defer.Deferred` objects.
- When :ref:`using the asyncio reactor <install-asyncio>`, you can only
await on :class:`asyncio.Future` objects.
If you want to write code that uses ``Deferred`` objects but works with any
reactor, use this function on all ``Deferred`` objects::
class MySpider(Spider):
...
async def parse(self, response):
d = treq.get('https://example.com/additional')
extra_response = await maybe_deferred_to_future(d)
"""
if not is_asyncio_reactor_installed():
return d
else:
return deferred_to_future(d)

8
scrapy/utils/log.py
View File

@ -124,8 +124,9 @@ def _get_handler(settings):
""" Return a log handler object according to settings """
filename = settings.get('LOG_FILE')
if filename:
mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
encoding = settings.get('LOG_ENCODING')
handler = logging.FileHandler(filename, encoding=encoding)
handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
elif settings.getbool('LOG_ENABLED'):
handler = logging.StreamHandler()
else:
@ -142,7 +143,7 @@ def _get_handler(settings):
return handler
def log_scrapy_info(settings):
def log_scrapy_info(settings: Settings) -> None:
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
versions = [
@ -151,6 +152,9 @@ def log_scrapy_info(settings):
if name != "Scrapy"
]
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
def log_reactor_info() -> None:
from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
from twisted.internet import asyncioreactor

Some files were not shown because too many files have changed in this diff Show More