mirror of https://github.com/scrapy/scrapy.git
fix
This commit is contained in:
Akshay Sharma
commit
3e9efe26ce
|
|
@ -8,6 +8,7 @@ skips:
|
|||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B324
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.5.0
|
||||
current_version = 2.6.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.9
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: security
|
||||
- python-version: 3.9
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: flake8
|
||||
# Pylint requires installing reppy, which does not support Python 3.9
|
||||
|
|
@ -19,11 +19,10 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
- python-version: 3.9
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: 3.8 # Keep in sync with .readthedocs.yml
|
||||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
|
||||
|
|
@ -38,8 +37,5 @@ jobs:
|
|||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python 3.9
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Check Tag
|
||||
id: check-release-tag
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [3.6, 3.7, 3.8, 3.9]
|
||||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -8,31 +8,34 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.3
|
||||
PYPY_VERSION: 3.9-v7.3.9
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: pinned
|
||||
- python-version: 3.6.12
|
||||
- python-version: 3.7.13
|
||||
env:
|
||||
TOXENV: asyncio-pinned
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3-pinned
|
||||
PYPY_VERSION: 3.6-v7.2.0
|
||||
PYPY_VERSION: 3.7-v7.3.5
|
||||
|
||||
# extras
|
||||
# extra-deps includes reppy, which does not support Python 3.9
|
||||
|
|
@ -40,15 +43,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
|
||||
# 3.10-pre
|
||||
- python-version: "3.10.0-beta.4"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10.0-beta.4"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -75,9 +69,6 @@ jobs:
|
|||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
|
|
|
|||
|
|
@ -8,19 +8,21 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.7
|
||||
env:
|
||||
TOXENV: py
|
||||
TOXENV: windows-pinned
|
||||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
# https://twistedmatrix.com/trac/ticket/9990
|
||||
#- python-version: 3.9
|
||||
#env:
|
||||
#TOXENV: py
|
||||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -5,12 +5,13 @@ sphinx:
|
|||
fail_on_warning: true
|
||||
|
||||
build:
|
||||
image: latest
|
||||
os: ubuntu-20.04
|
||||
tools:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
|
||||
python: "3.10" # Keep in sync with .github/workflows/checks.yml
|
||||
|
||||
python:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
|
||||
version: 3.8 # Keep in sync with .github/workflows/checks.yml
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- path: .
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. image:: https://scrapy.org/img/scrapylogo.png
|
||||
|
||||
======
|
||||
Scrapy
|
||||
======
|
||||
|
|
@ -55,7 +57,7 @@ including a list of features.
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.6+
|
||||
* Python 3.7+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
|
|
@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest):
|
|||
pytest.skip('This test is only run with --reactor=asyncio')
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def only_not_asyncio(request, reactor_pytest):
|
||||
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
|
||||
pytest.skip('This test is only run without --reactor=asyncio')
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
if config.getoption("--reactor") == "asyncio":
|
||||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
|
|||
Start over
|
||||
----------
|
||||
|
||||
To cleanup all generated documentation files and start from scratch run::
|
||||
To clean up all generated documentation files and start from scratch run::
|
||||
|
||||
make clean
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<base href='http://example.com/' />
|
||||
<title>Example website</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id='images'>
|
||||
<a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
|
||||
<a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
|
||||
<a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
|
||||
<a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
|
||||
<a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
|
|||
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
|
||||
|
||||
# Never documented before, and deprecated now.
|
||||
r'^scrapy\.item\.DictItem$',
|
||||
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
|
||||
|
||||
# Implementation detail of LxmlLinkExtractor
|
||||
|
|
@ -295,7 +294,9 @@ intersphinx_mapping = {
|
|||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
|
||||
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
|
||||
}
|
||||
intersphinx_disabled_reftypes = []
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
|
|
@ -304,10 +305,14 @@ intersphinx_mapping = {
|
|||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"command": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
"reqmeta": "tooltip",
|
||||
"setting": "tooltip",
|
||||
"signal": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
|||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
try:
|
||||
# >2.0.1
|
||||
from sybil.parsers.codeblock import PythonCodeBlockParser
|
||||
except ImportError:
|
||||
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
|
@ -21,7 +25,7 @@ def setup(namespace):
|
|||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
PythonCodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
|
|
|
|||
|
|
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
the tests with Python 3.7 use::
|
||||
|
||||
tox -e py36
|
||||
tox -e py37
|
||||
|
||||
You can also specify a comma-separated list of environments, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py36,py38 -p auto
|
||||
tox -e py37,py38 -p auto
|
||||
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
|
|
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
|
|||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
tox -e py37 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
|
|
|||
|
|
@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
|
|||
|
||||
See :ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
|
||||
Scrapy crashes with: ImportError: No module named win32api
|
||||
----------------------------------------------------------
|
||||
|
||||
You need to install `pywin32`_ because of `this Twisted bug`_.
|
||||
|
||||
.. _pywin32: https://sourceforge.net/projects/pywin32/
|
||||
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
|
||||
|
||||
How can I simulate a user login in my spider?
|
||||
---------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ testing.
|
|||
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
|
||||
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
|
||||
|
||||
.. _getting-help:
|
||||
|
||||
Getting help
|
||||
============
|
||||
|
||||
|
|
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
|
|||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
|
||||
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
|
||||
|
||||
|
||||
First steps
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Examples
|
|||
The best way to learn is with examples, and Scrapy is no exception. For this
|
||||
reason, there is an example Scrapy project named quotesbot_, that you can use to
|
||||
play and learn more about Scrapy. It contains two spiders for
|
||||
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
expressions.
|
||||
|
||||
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ Installation guide
|
|||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.6+, either the CPython implementation (default) or
|
||||
the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
|
||||
Scrapy requires Python 3.7+, either the CPython implementation (default) or
|
||||
the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
.. _intro-install-scrapy:
|
||||
|
||||
|
|
@ -52,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
|
|||
* `twisted`_, an asynchronous networking framework
|
||||
* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
|
||||
|
||||
The minimal versions which Scrapy is tested against are:
|
||||
|
||||
* Twisted 14.0
|
||||
* lxml 3.4
|
||||
* pyOpenSSL 0.14
|
||||
|
||||
Scrapy may work with older versions of these packages
|
||||
but it is not guaranteed it will continue working
|
||||
because it’s not being tested against them.
|
||||
|
||||
Some of these packages themselves depends on non-Python packages
|
||||
that might require additional installation steps depending on your platform.
|
||||
Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
|
||||
|
|
@ -190,7 +180,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
|
|||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
* *(Recommended)* **Don't** use system Python. Install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
|
|
@ -234,8 +224,8 @@ For PyPy3, only Linux installation was tested.
|
|||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependencies will be built during installation.
|
||||
On macOS, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
On macOS, you are likely to face an issue with building the Cryptography
|
||||
dependency. The solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing Scrapy). Installing on Linux has no special
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
Scrapy at a glance
|
||||
==================
|
||||
|
||||
Scrapy is an application framework for crawling web sites and extracting
|
||||
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
|
||||
structured data which can be used for a wide range of useful applications, like
|
||||
data mining, information processing or historical archival.
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
|
|||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
https://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
'https://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -45,9 +45,9 @@ http://quotes.toscrape.com, following the pagination::
|
|||
Put this in a text file, name it to something like ``quotes_spider.py``
|
||||
and run the spider using the :command:`runspider` command::
|
||||
|
||||
scrapy runspider quotes_spider.py -o quotes.jl
|
||||
scrapy runspider quotes_spider.py -o quotes.jsonl
|
||||
|
||||
When this finishes you will have in the ``quotes.jl`` file a list of the
|
||||
When this finishes you will have in the ``quotes.jsonl`` file a list of the
|
||||
quotes in JSON Lines format, containing text and author, looking like this::
|
||||
|
||||
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Scrapy Tutorial
|
|||
In this tutorial, we'll assume that Scrapy is already installed on your system.
|
||||
If that's not the case, see :ref:`intro-install`.
|
||||
|
||||
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
|
||||
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
|
||||
that lists quotes from famous authors.
|
||||
|
||||
This tutorial will walk you through these tasks:
|
||||
|
|
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
|
|||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
|
@ -143,9 +143,9 @@ similar to this::
|
|||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
|
@ -184,8 +184,8 @@ for your spider::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -207,7 +207,7 @@ Extracting data
|
|||
The best way to learn how to extract data with Scrapy is trying selectors
|
||||
using the :ref:`Scrapy shell <topics-shell>`. Run::
|
||||
|
||||
scrapy shell 'http://quotes.toscrape.com/page/1/'
|
||||
scrapy shell 'https://quotes.toscrape.com/page/1/'
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "http://quotes.toscrape.com/page/1/"
|
||||
scrapy shell "https://quotes.toscrape.com/page/1/"
|
||||
|
||||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 http://quotes.toscrape.com/page/1/>
|
||||
[s] request <GET https://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 https://quotes.toscrape.com/page/1/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
|
||||
[s] Useful shortcuts:
|
||||
|
|
@ -241,7 +241,7 @@ object:
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
|
@ -277,9 +277,19 @@ As an alternative, you could've written:
|
|||
>>> response.css('title::text')[0].get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
|
||||
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
|
||||
find any element matching the selection.
|
||||
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
|
||||
raise an :exc:`IndexError` exception if there are no results::
|
||||
|
||||
>>> response.css('noelement')[0].get()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
IndexError: list index out of range
|
||||
|
||||
You might want to use ``.get()`` directly on the
|
||||
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
|
||||
if there are no results::
|
||||
|
||||
>>> response.css("noelement").get()
|
||||
|
||||
There's a lesson here: for most scraping code, you want it to be resilient to
|
||||
errors due to things not being found on a page, so that even if some parts fail
|
||||
|
|
@ -345,7 +355,7 @@ Extracting quotes and authors
|
|||
Now that you know a bit about selection and extraction, let's complete our
|
||||
spider by writing the code to extract the quotes from the web page.
|
||||
|
||||
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
|
||||
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
|
||||
like this:
|
||||
|
||||
.. code-block:: html
|
||||
|
|
@ -369,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'http://quotes.toscrape.com'
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
@ -434,8 +444,8 @@ in the callback, as you can see below::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -448,9 +458,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
@ -472,13 +482,13 @@ to append new content to any existing file. However, appending to a JSON file
|
|||
makes the file contents invalid JSON. When appending to a file, consider
|
||||
using a different serialization format, such as `JSON Lines`_::
|
||||
|
||||
scrapy crawl quotes -o quotes.jl
|
||||
scrapy crawl quotes -o quotes.jsonl
|
||||
|
||||
The `JSON Lines`_ format is useful because it's stream-like, you can easily
|
||||
append new records to it. It doesn't have the same problem of JSON when you run
|
||||
twice. Also, as each record is a separate line, you can process big files
|
||||
without having to fit everything in memory, there are tools like `JQ`_ to help
|
||||
doing that at the command-line.
|
||||
do that at the command-line.
|
||||
|
||||
In small projects (like the one in this tutorial), that should be enough.
|
||||
However, if you want to perform more complex things with the scraped items, you
|
||||
|
|
@ -495,7 +505,7 @@ Following links
|
|||
===============
|
||||
|
||||
Let's say, instead of just scraping the stuff from the first two pages
|
||||
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
|
||||
Now that you know how to extract data from pages, let's see how to follow links
|
||||
from them.
|
||||
|
|
@ -539,7 +549,7 @@ page, extracting data from it::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -644,7 +654,7 @@ this time for scraping author information::
|
|||
class AuthorSpider(scrapy.Spider):
|
||||
name = 'author'
|
||||
|
||||
start_urls = ['http://quotes.toscrape.com/']
|
||||
start_urls = ['https://quotes.toscrape.com/']
|
||||
|
||||
def parse(self, response):
|
||||
author_page_links = response.css('.author + a')
|
||||
|
|
@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument::
|
|||
name = "quotes"
|
||||
|
||||
def start_requests(self):
|
||||
url = 'http://quotes.toscrape.com/'
|
||||
url = 'https://quotes.toscrape.com/'
|
||||
tag = getattr(self, 'tag', None)
|
||||
if tag is not None:
|
||||
url = url + 'tag/' + tag
|
||||
|
|
@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument::
|
|||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
will only visit URLs from the ``humor`` tag, such as
|
||||
``http://quotes.toscrape.com/tag/humor``.
|
||||
``https://quotes.toscrape.com/tag/humor``.
|
||||
|
||||
You can :ref:`learn more about handling spider arguments here <spiderargs>`.
|
||||
|
||||
|
|
|
|||
545
docs/news.rst
545
docs/news.rst
|
|
@ -3,6 +3,450 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.6.1:
|
||||
|
||||
Scrapy 2.6.1 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Fixes a regression introduced in 2.6.0 that would unset the request method when
|
||||
following redirects.
|
||||
|
||||
|
||||
.. _release-2.6.0:
|
||||
|
||||
Scrapy 2.6.0 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Security fixes for cookie handling <2.6-security-fixes>`
|
||||
|
||||
* Python 3.10 support
|
||||
|
||||
* :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental, and works out-of-the-box on Windows regardless of your Python
|
||||
version
|
||||
|
||||
* Feed exports now support :class:`pathlib.Path` output paths and per-feed
|
||||
:ref:`item filtering <item-filter>` and
|
||||
:ref:`post-processing <post-processing>`
|
||||
|
||||
.. _2.6-security-fixes:
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies from a
|
||||
controlled domain into your cookiejar that could be sent to other domains
|
||||
not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
|
||||
advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The h2_ dependency is now optional, only needed to
|
||||
:ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
|
||||
|
||||
.. _h2: https://pypi.org/project/h2/
|
||||
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
|
||||
for a non-POST request, now overrides the URL query string, instead of
|
||||
being appended to it. (:issue:`2919`, :issue:`3579`)
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
|
||||
the return value of that function, and not the ``params`` input parameter,
|
||||
will determine the feed URI parameters, unless that return value is
|
||||
``None``. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- In :class:`scrapy.core.engine.ExecutionEngine`, methods
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
|
||||
and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
now raise :exc:`RuntimeError` if called before
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
|
||||
|
||||
These methods used to assume that
|
||||
:attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
|
||||
been defined by a prior call to
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
|
||||
raising :exc:`AttributeError` instead.
|
||||
|
||||
- If the API of the configured :ref:`scheduler <topics-scheduler>` does not
|
||||
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
|
||||
other exceptions would be raised at run time. (:issue:`3559`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
|
||||
now been removed. (:issue:`5393`)
|
||||
|
||||
- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
|
||||
been removed. (:issue:`4178`, :issue:`4356`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
|
||||
returning ``None`` or modifying the ``params`` input parameter is now
|
||||
deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
|
||||
:meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
|
||||
:func:`scrapy.utils.request.request_from_dict` function.
|
||||
|
||||
- In :mod:`scrapy.squeues`, the following queue classes are deprecated:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
|
||||
instead use:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueue`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
|
||||
|
||||
- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
|
||||
a time when this class could handle multiple :class:`~scrapy.Spider`
|
||||
objects at a time have been deprecated. (:issue:`5090`)
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
|
||||
is deprecated.
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
|
||||
deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
|
||||
|
||||
- The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
|
||||
is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
|
||||
instead.
|
||||
|
||||
- The ``spider`` parameter is deprecated for the following methods:
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.download`
|
||||
|
||||
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
|
||||
first to set the :class:`~scrapy.Spider` object.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- You can now use :ref:`item filtering <item-filter>` to control which items
|
||||
are exported to each output feed. (:issue:`4575`, :issue:`5178`,
|
||||
:issue:`5161`, :issue:`5203`)
|
||||
|
||||
- You can now apply :ref:`post-processing <post-processing>` to feeds, and
|
||||
:ref:`built-in post-processing plugins <builtin-plugins>` are provided for
|
||||
output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
|
||||
|
||||
- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
|
||||
keys. (:issue:`5383`, :issue:`5384`)
|
||||
|
||||
- Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
|
||||
or later will automatically switch the asyncio event loop to one that
|
||||
allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
|
||||
:issue:`5315`)
|
||||
|
||||
- The :command:`genspider` command now supports a start URL instead of a
|
||||
domain name. (:issue:`4439`)
|
||||
|
||||
- :mod:`scrapy.utils.defer` gained 2 new functions,
|
||||
:func:`~scrapy.utils.defer.deferred_to_future` and
|
||||
:func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
|
||||
on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
|
||||
(:issue:`5288`)
|
||||
|
||||
- :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
|
||||
support for `temporary security credentials`_
|
||||
(:setting:`AWS_SESSION_TOKEN`) and endpoint customization
|
||||
(:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
|
||||
(:issue:`5279`)
|
||||
|
||||
- :attr:`Request.cookies <scrapy.Request.cookies>` values that are
|
||||
:class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
|
||||
(:issue:`5252`, :issue:`5253`)
|
||||
|
||||
- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
|
||||
the :signal:`spider_idle` signal to customize the reason why the spider is
|
||||
stopping. (:issue:`5191`)
|
||||
|
||||
- When using
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
|
||||
proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
|
||||
scheme. (:issue:`4505`, :issue:`4649`)
|
||||
|
||||
- All built-in queues now expose a ``peek`` method that returns the next
|
||||
queue object (like ``pop``) but does not remove the returned object from
|
||||
the queue. (:issue:`5112`)
|
||||
|
||||
If the underlying queue does not support peeking (e.g. because you are not
|
||||
using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
|
||||
:exc:`NotImplementedError`.
|
||||
|
||||
- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
|
||||
an ``attributes`` attribute that makes subclassing easier. For
|
||||
:class:`~scrapy.http.Request`, it also allows subclasses to work with
|
||||
:func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
|
||||
:issue:`5130`, :issue:`5218`)
|
||||
|
||||
- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
|
||||
:meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
|
||||
:ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
|
||||
|
||||
- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
|
||||
exceptions now only truncate response bodies longer than 1000 characters,
|
||||
instead of those longer than 32 characters, making it easier to debug such
|
||||
errors. (:issue:`4881`, :issue:`5007`)
|
||||
|
||||
- :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
|
||||
(:issue:`5145`, :issue:`5269`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
|
||||
are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
|
||||
(:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Removed a module-level Twisted reactor import that could prevent
|
||||
:ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
|
||||
|
||||
- The :command:`startproject` command works with existing folders again.
|
||||
(:issue:`4665`, :issue:`4676`)
|
||||
|
||||
- The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
|
||||
(:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
|
||||
``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
|
||||
|
||||
- Made :func:`scrapy.utils.response.open_in_browser` support more complex
|
||||
HTML. (:issue:`5319`, :issue:`5320`)
|
||||
|
||||
- Fixed :attr:`CSVFeedSpider.quotechar
|
||||
<scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
|
||||
encoding. (:issue:`5391`, :issue:`5394`)
|
||||
|
||||
- Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
|
||||
|
||||
.. _setuptools: https://pypi.org/project/setuptools/
|
||||
|
||||
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
now also works as expected with links that have comma-separated ``rel``
|
||||
attribute values including ``nofollow``. (:issue:`5225`)
|
||||
|
||||
- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
|
||||
<topics-feed-exports>` parameter parsing. (:issue:`5359`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental. (:issue:`5332`)
|
||||
|
||||
- Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
|
||||
(:issue:`4976`, :issue:`5315`)
|
||||
|
||||
- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
|
||||
(:issue:`4484`, :issue:`4613`)
|
||||
|
||||
- Documented :ref:`local file naming in media pipelines
|
||||
<topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
|
||||
|
||||
- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
|
||||
:issue:`3669`)
|
||||
|
||||
- Provided better context and instructions to disable the
|
||||
:setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
|
||||
|
||||
- Documented that :ref:`reppy-parser` does not support Python 3.9+.
|
||||
(:issue:`5226`, :issue:`5231`)
|
||||
|
||||
- Documented :ref:`the scheduler component <topics-scheduler>`.
|
||||
(:issue:`3537`, :issue:`3559`)
|
||||
|
||||
- Documented the method used by :ref:`media pipelines
|
||||
<topics-media-pipeline>` to :ref:`determine if a file has expired
|
||||
<file-expiration>`. (:issue:`5120`, :issue:`5254`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now features
|
||||
:func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now covers what happens when you define
|
||||
different per-spider values for some settings that cannot differ at run
|
||||
time. (:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Extended the documentation of the
|
||||
:class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
|
||||
(:issue:`5199`, :issue:`5217`)
|
||||
|
||||
- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
|
||||
:issue:`5224`)
|
||||
|
||||
- Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
|
||||
(:issue:`5174`, :issue:`5244`)
|
||||
|
||||
- Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
|
||||
(:issue:`1582`)
|
||||
|
||||
- Added the ``body_length`` parameter to the documented signature of the
|
||||
:signal:`headers_received` signal. (:issue:`5270`)
|
||||
|
||||
- Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
|
||||
in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
|
||||
|
||||
- The documentation now features the shortest import path of classes with
|
||||
multiple import paths. (:issue:`2733`, :issue:`5099`)
|
||||
|
||||
- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
|
||||
(:issue:`5395`, :issue:`5396`)
|
||||
|
||||
- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
|
||||
to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
|
||||
|
||||
- The pronunciation of the project name is now :ref:`officially
|
||||
<intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
|
||||
|
||||
- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
|
||||
|
||||
- Fixed issues and implemented minor improvements. (:issue:`3155`,
|
||||
:issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
|
||||
:issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
|
||||
:issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
|
||||
:issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
|
||||
|
||||
|
||||
Quality Assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
|
||||
:issue:`5265`)
|
||||
|
||||
- Significantly reduced memory usage by
|
||||
:func:`scrapy.utils.response.response_httprepr`, used by the
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
|
||||
middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
|
||||
|
||||
- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
|
||||
:issue:`5374`)
|
||||
|
||||
- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
|
||||
:issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
|
||||
|
||||
- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
|
||||
:issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
|
||||
:issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
|
||||
:issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
|
||||
:issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
|
||||
:issue:`5425`, :issue:`5427`)
|
||||
|
||||
- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
|
||||
:issue:`5177`, :issue:`5200`)
|
||||
|
||||
- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
|
||||
:issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
|
||||
:issue:`5314`, :issue:`5322`)
|
||||
|
||||
|
||||
.. _release-2.5.1:
|
||||
|
||||
Scrapy 2.5.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-2.5.0:
|
||||
|
||||
Scrapy 2.5.0 (2021-04-06)
|
||||
|
|
@ -940,9 +1384,8 @@ Bug fixes
|
|||
* zope.interface 5.0.0 and later versions are now supported
|
||||
(:issue:`4447`, :issue:`4448`)
|
||||
|
||||
* :meth:`Spider.make_requests_from_url
|
||||
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
|
||||
1.4.0, now issues a warning when used (:issue:`4412`)
|
||||
* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
|
||||
warning when used (:issue:`4412`)
|
||||
|
||||
|
||||
Documentation
|
||||
|
|
@ -1200,7 +1643,7 @@ New features
|
|||
:issue:`4370`)
|
||||
|
||||
* A new ``keep_fragments`` parameter of
|
||||
:func:`scrapy.utils.request.request_fingerprint` allows to generate
|
||||
``scrapy.utils.request.request_fingerprint`` allows to generate
|
||||
different fingerprints for requests with different fragments in their URL
|
||||
(:issue:`4104`)
|
||||
|
||||
|
|
@ -1454,6 +1897,88 @@ affect subclasses:
|
|||
(:issue:`3884`)
|
||||
|
||||
|
||||
.. _release-1.8.2:
|
||||
|
||||
Scrapy 1.8.2 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
**Security bug fixes:**
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies into
|
||||
your requests to some other domains. Please, see the `mfjm-vh54-3f96
|
||||
security advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
.. _release-1.8.1:
|
||||
|
||||
Scrapy 1.8.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
|
|
@ -1754,7 +2279,7 @@ New features
|
|||
* A new scheduler priority queue,
|
||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targetting multiple web domains, at the
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
|
||||
|
|
@ -2792,7 +3317,7 @@ Bug fixes
|
|||
- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
|
||||
(:issue:`2225`).
|
||||
- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
|
||||
- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
|
||||
Refactoring
|
||||
~~~~~~~~~~~
|
||||
|
|
@ -3655,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09)
|
|||
- adding some xpath tips to selectors docs (:commit:`2d103e0`)
|
||||
- fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`)
|
||||
- get_func_args maximum recursion fix #728 (:commit:`81344ea`)
|
||||
- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Updated input/output processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Fixed Python syntax in tutorial. (:commit:`db59ed9`)
|
||||
- Add test case for tunneling proxy (:commit:`f090260`)
|
||||
- Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`)
|
||||
|
|
@ -4282,7 +4807,7 @@ Code rearranged and removed
|
|||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
|
|
@ -4317,7 +4842,7 @@ Scrapyd changes
|
|||
~~~~~~~~~~~~~~~
|
||||
|
||||
- Scrapyd now uses one process per spider
|
||||
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
|
||||
- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default)
|
||||
- A minimal web ui was added, available at http://localhost:6800 by default
|
||||
- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
|
||||
|
||||
|
|
@ -4353,7 +4878,7 @@ New features and improvements
|
|||
- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
|
||||
- Support for overriding default request headers per spider (#181)
|
||||
- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
|
||||
- Splitted Debian package into two packages - the library and the service (#187)
|
||||
- Split Debian package into two packages - the library and the service (#187)
|
||||
- Scrapy log refactoring (#188)
|
||||
- New extension for keeping persistent spider contexts among different runs (#203)
|
||||
- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
|
|||
:class:`scrapy.Spider` subclass and a
|
||||
:class:`scrapy.settings.Settings` object.
|
||||
|
||||
.. attribute:: request_fingerprinter
|
||||
|
||||
The request fingerprint builder of this crawler.
|
||||
|
||||
This is used from extensions and middlewares to build short, unique
|
||||
identifiers for requests. See :ref:`request-fingerprints`.
|
||||
|
||||
.. attribute:: settings
|
||||
|
||||
The settings manager of this crawler.
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ this:
|
|||
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
|
||||
to crawl.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
9. The process repeats (from step 3) until there are no more requests from the
|
||||
:ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
Components
|
||||
|
|
|
|||
|
|
@ -10,10 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
|||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
|
||||
recommended for production environments. Future Scrapy versions
|
||||
may introduce related changes without a deprecation period or
|
||||
warning.
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
|
|
@ -30,6 +26,7 @@ reactor manually. You can do that using
|
|||
|
||||
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
|
||||
|
||||
|
||||
.. _using-custom-loops:
|
||||
|
||||
Using custom asyncio loops
|
||||
|
|
@ -40,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
|
|||
use it instead of the default asyncio event loop.
|
||||
|
||||
|
||||
.. _asyncio-windows:
|
||||
|
||||
Windows-specific notes
|
||||
======================
|
||||
|
||||
The Windows implementation of :mod:`asyncio` can use two event loop
|
||||
implementations:
|
||||
|
||||
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
|
||||
when using Twisted.
|
||||
|
||||
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
|
||||
with Twisted.
|
||||
|
||||
So on Python 3.8+ the event loop class needs to be changed.
|
||||
|
||||
.. versionchanged:: 2.6.0
|
||||
The event loop class is changed automatically when you change the
|
||||
:setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
To change the event loop class manually, call the following code before
|
||||
installing the reactor::
|
||||
|
||||
import asyncio
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
You can put this in the same function that installs the reactor, if you do that
|
||||
yourself, or in some code that runs before the reactor is installed, e.g.
|
||||
``settings.py``.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
subprocesses (this is the case with `playwright`_), so you cannot use
|
||||
them together with Scrapy on Windows (but you should be able to use
|
||||
them on WSL or native Linux).
|
||||
|
||||
.. _playwright: https://github.com/microsoft/playwright-python
|
||||
|
||||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
Awaiting on Deferreds
|
||||
=====================
|
||||
|
||||
When the asyncio reactor isn't installed, you can await on Deferreds in the
|
||||
coroutines directly. When it is installed, this is not possible anymore, due to
|
||||
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
|
||||
:class:`asyncio.Future` objects, not into
|
||||
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
|
||||
Futures. Scrapy provides two helpers for this:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_to_future
|
||||
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
|
||||
.. tip:: If you need to use these functions in code that aims to be compatible
|
||||
with lower versions of Scrapy that do not provide these functions,
|
||||
down to Scrapy 2.0 (earlier versions do not support
|
||||
:mod:`asyncio`), you can copy the implementation of these functions
|
||||
into your own code.
|
||||
|
|
|
|||
|
|
@ -230,10 +230,16 @@ Usage example::
|
|||
genspider
|
||||
---------
|
||||
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
|
||||
* Requires project: *no*
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
.. versionadded:: 2.6.0
|
||||
The ability to pass a URL instead of a domain.
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
|
||||
.. note:: Even if an HTTPS URL is specified, the protocol used in
|
||||
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. _topics-coroutines:
|
||||
|
||||
==========
|
||||
Coroutines
|
||||
==========
|
||||
|
|
@ -75,23 +77,28 @@ coroutines, functions that return Deferreds and functions that return
|
|||
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
|
||||
This means you can use many useful Python libraries providing such code::
|
||||
|
||||
class MySpider(Spider):
|
||||
class MySpiderDeferred(Spider):
|
||||
# ...
|
||||
async def parse_with_deferred(self, response):
|
||||
async def parse(self, response):
|
||||
additional_response = await treq.get('https://additional.url')
|
||||
additional_data = await treq.content(additional_response)
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
async def parse_with_asyncio(self, response):
|
||||
class MySpiderAsyncio(Spider):
|
||||
# ...
|
||||
async def parse(self, response):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get('https://additional.url') as additional_response:
|
||||
additional_data = await r.text()
|
||||
additional_data = await additional_response.text()
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
|
||||
:mod:`asyncio` loop and to use them you need to
|
||||
:doc:`enable asyncio support in Scrapy<asyncio>`.
|
||||
|
||||
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
|
||||
you need to :ref:`wrap them<asyncio-await-dfd>`.
|
||||
|
||||
Common use cases for asynchronous code include:
|
||||
|
||||
* requesting data from websites, databases and other services (in callbacks,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
|
|||
|
||||
Since Developer Tools operate on a live browser DOM, what you'll actually see
|
||||
when inspecting the page source is not the original HTML, but a modified one
|
||||
after applying some browser clean up and executing Javascript code. Firefox,
|
||||
after applying some browser clean up and executing JavaScript code. Firefox,
|
||||
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
|
||||
the other hand, does not modify the original page HTML, so you won't be able to
|
||||
extract any data if you use ``<tbody>`` in your XPath expressions.
|
||||
|
||||
Therefore, you should keep in mind the following things:
|
||||
|
||||
* Disable Javascript while inspecting the DOM looking for XPaths to be
|
||||
* Disable JavaScript while inspecting the DOM looking for XPaths to be
|
||||
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
|
||||
|
||||
* Never use full XPath paths, use relative and clever ones based on attributes
|
||||
|
|
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
|
|||
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
|
||||
copy XPaths to selected elements. Let's try it out.
|
||||
|
||||
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
|
||||
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
$ scrapy shell "http://quotes.toscrape.com/"
|
||||
$ scrapy shell "https://quotes.toscrape.com/"
|
||||
|
||||
Then, back to your web browser, right-click on the ``span`` tag, select
|
||||
``Copy > XPath`` and paste it in the Scrapy shell like so:
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
|
||||
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
|
||||
|
||||
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
|
||||
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
|
||||
|
|
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
|
|||
type ``json``.
|
||||
|
||||
If we click on this request, we see that the request URL is
|
||||
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
is a JSON-object that contains our quotes. We can also right-click
|
||||
on the request and open ``Open in new tab`` to get a better overview.
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
|
|||
name = 'quote'
|
||||
allowed_domains = ['quotes.toscrape.com']
|
||||
page = 1
|
||||
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
|
||||
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
|
||||
|
||||
def parse(self, response):
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
|
|||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
self.page += 1
|
||||
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
This spider starts at the first page of the quotes-API. With each
|
||||
|
|
@ -280,7 +280,7 @@ request::
|
|||
from scrapy import Request
|
||||
|
||||
request = Request.from_curl(
|
||||
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
|
||||
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
|
||||
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
|
||||
|
|
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
|
|||
to identifying the correct request and replicating it in your spider.
|
||||
|
||||
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
|
||||
.. _quotes.toscrape.com: http://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _quotes.toscrape.com: https://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
methods of installed middleware is always called on every response.
|
||||
|
||||
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
|
||||
process_request methods and reschedule the returned request. Once the newly returned
|
||||
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
|
||||
request is performed, the appropriate middleware chain will be called on
|
||||
the downloaded response.
|
||||
|
||||
|
|
@ -323,8 +323,21 @@ HttpAuthMiddleware
|
|||
This middleware authenticates all requests generated from certain spiders
|
||||
using `Basic access authentication`_ (aka. HTTP auth).
|
||||
|
||||
To enable HTTP authentication from certain spiders, set the ``http_user``
|
||||
and ``http_pass`` attributes of those spiders.
|
||||
To enable HTTP authentication for a spider, set the ``http_user`` and
|
||||
``http_pass`` spider attributes to the authentication data and the
|
||||
``http_auth_domain`` spider attribute to the domain which requires this
|
||||
authentication (its subdomains will be also handled in the same way).
|
||||
You can set ``http_auth_domain`` to ``None`` to enable the
|
||||
authentication for all requests but you risk leaking your authentication
|
||||
credentials to unrelated domains.
|
||||
|
||||
.. warning::
|
||||
In previous Scrapy versions HttpAuthMiddleware sent the authentication
|
||||
data with all requests, which is a security problem if the spider
|
||||
makes requests to several different domains. Currently if the
|
||||
``http_auth_domain`` attribute is not set, the middleware will use the
|
||||
domain of the first request, which will work for some spiders but not
|
||||
for others. In the future the middleware will produce an error instead.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -334,6 +347,7 @@ HttpAuthMiddleware
|
|||
|
||||
http_user = 'someuser'
|
||||
http_pass = 'somepass'
|
||||
http_auth_domain = 'intranet.example.com'
|
||||
name = 'intranet.example.com'
|
||||
|
||||
# .. rest of the spider code omitted ...
|
||||
|
|
@ -352,7 +366,7 @@ HttpCacheMiddleware
|
|||
This middleware provides low-level cache to all HTTP requests and responses.
|
||||
It has to be combined with a cache storage backend as well as a cache policy.
|
||||
|
||||
Scrapy ships with three HTTP cache storage backends:
|
||||
Scrapy ships with the following HTTP cache storage backends:
|
||||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
|
|
@ -690,14 +704,15 @@ HttpCompressionMiddleware
|
|||
sent/received from web sites.
|
||||
|
||||
This middleware also supports decoding `brotli-compressed`_ as well as
|
||||
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
|
||||
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
|
||||
installed, respectively.
|
||||
|
||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||
.. _brotlipy: https://pypi.org/project/brotlipy/
|
||||
.. _brotli: https://pypi.org/project/Brotli/
|
||||
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
||||
.. _zstandard: https://pypi.org/project/zstandard/
|
||||
|
||||
|
||||
HttpCompressionMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -1005,7 +1020,7 @@ Parsers vary in several aspects:
|
|||
(shorter) rule
|
||||
|
||||
Performance comparison of different parsers is available at `the following link
|
||||
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
|
||||
<https://github.com/scrapy/scrapy/issues/3969>`_.
|
||||
|
||||
.. _protego-parser:
|
||||
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ Example::
|
|||
class ProductXmlExporter(XmlItemExporter):
|
||||
|
||||
def serialize_field(self, field, name, value):
|
||||
if field == 'price':
|
||||
if name == 'price':
|
||||
return f'$ {str(value)}'
|
||||
return super().serialize_field(field, name, value)
|
||||
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ feed URI, allowing item delivery to start way before the end of the crawl.
|
|||
Item filtering
|
||||
==============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
You can filter items that you want to allow for a particular feed by using the
|
||||
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
|
||||
|
|
@ -318,11 +318,11 @@ ItemFilter
|
|||
Post-Processing
|
||||
===============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
|
|
@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods:
|
|||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
|
|
@ -457,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
If undefined or empty, all items are exported.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
|
||||
|
||||
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
|
|
@ -499,7 +499,7 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
The plugins will be used in the order of the list passed.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -638,6 +638,7 @@ Default::
|
|||
{
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.exporters.CsvItemExporter',
|
||||
'xml': 'scrapy.exporters.XmlItemExporter',
|
||||
|
|
@ -744,6 +745,9 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
|
|
@ -760,7 +764,7 @@ source spider in the feed URI:
|
|||
|
||||
#. Use ``%(spider_name)s`` in your feed URI::
|
||||
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
|
||||
|
||||
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
|
|
|
|||
|
|
@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
|
|||
:param spider: the spider which was closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a pipeline instance
|
||||
If present, this class method is called to create a pipeline instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the pipeline. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for pipeline to
|
||||
|
|
@ -99,11 +99,11 @@ contain a price::
|
|||
raise DropItem(f"Missing price in {item}")
|
||||
|
||||
|
||||
Write items to a JSON file
|
||||
--------------------------
|
||||
Write items to a JSON lines file
|
||||
--------------------------------
|
||||
|
||||
The following pipeline stores all scraped items (from all spiders) into a
|
||||
single ``items.jl`` file, containing one item per line serialized in JSON
|
||||
single ``items.jsonl`` file, containing one item per line serialized in JSON
|
||||
format::
|
||||
|
||||
import json
|
||||
|
|
@ -113,7 +113,7 @@ format::
|
|||
class JsonWriterPipeline:
|
||||
|
||||
def open_spider(self, spider):
|
||||
self.file = open('items.jl', 'w')
|
||||
self.file = open('items.jsonl', 'w')
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.file.close()
|
||||
|
|
@ -190,6 +190,8 @@ item.
|
|||
|
||||
import scrapy
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
|
||||
class ScreenshotPipeline:
|
||||
"""Pipeline that uses Splash to render screenshot of
|
||||
|
|
@ -202,7 +204,7 @@ item.
|
|||
encoded_item_url = quote(adapter["url"])
|
||||
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
|
||||
request = scrapy.Request(screenshot_url)
|
||||
response = await spider.crawler.engine.download(request, spider)
|
||||
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
|
||||
|
||||
if response.status != 200:
|
||||
# Error happened, return item.
|
||||
|
|
|
|||
|
|
@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
|
|||
* define custom field metadata through :func:`dataclasses.field`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
They work natively in Python 3.7 or later, or using the `dataclasses
|
||||
backport`_ in Python 3.6.
|
||||
|
||||
.. _dataclasses backport: https://pypi.org/project/dataclasses/
|
||||
|
||||
Example::
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
|
|||
|
||||
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
|
||||
|
||||
.. _topics-keeping-persistent-state-between-batches:
|
||||
|
||||
Keeping persistent state between batches
|
||||
========================================
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ chapter <topics-items>`::
|
|||
l.add_xpath('name', '//div[@class="product_name"]')
|
||||
l.add_xpath('name', '//div[@class="product_title"]')
|
||||
l.add_xpath('price', '//p[@id="price"]')
|
||||
l.add_css('stock', 'p#stock]')
|
||||
l.add_css('stock', 'p#stock')
|
||||
l.add_value('last_updated', 'today') # you can also use literal values
|
||||
return l.load_item()
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ Logging settings
|
|||
These settings can be used to configure the logging:
|
||||
|
||||
* :setting:`LOG_FILE`
|
||||
* :setting:`LOG_FILE_APPEND`
|
||||
* :setting:`LOG_ENABLED`
|
||||
* :setting:`LOG_ENCODING`
|
||||
* :setting:`LOG_LEVEL`
|
||||
|
|
@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
|
|||
:setting:`LOG_FILE` is set, messages sent through the root logger will be
|
||||
redirected to a file named :setting:`LOG_FILE` with encoding
|
||||
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
|
||||
messages will be displayed on the standard error. Lastly, if
|
||||
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
|
||||
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
|
||||
(discarding the output from previous runs, if any). Lastly, if
|
||||
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
|
||||
|
||||
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
|
||||
|
|
@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many
|
|||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
|||
can configure some extra functions like generating thumbnails and filtering
|
||||
the images based on their size.
|
||||
|
||||
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
|
||||
The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
|
||||
thumbnailing and normalizing images to JPEG/RGB format.
|
||||
|
||||
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||
|
|
@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
|||
Additional features
|
||||
===================
|
||||
|
||||
.. _file-expiration:
|
||||
|
||||
File expiration
|
||||
---------------
|
||||
|
||||
|
|
@ -383,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
|
|||
|
||||
and pipeline class MyPipeline will have expiration time set to 180.
|
||||
|
||||
The last modified time from the file is used to determine the age of the file in days,
|
||||
which is then compared to the set expiration time to determine if the file is expired.
|
||||
|
||||
.. _topics-images-thumbnails:
|
||||
|
||||
Thumbnail generation for images
|
||||
|
|
@ -651,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
.. versionadded:: 2.4
|
||||
The *item* parameter.
|
||||
|
||||
.. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the
|
||||
thumbnail download path of the image originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>`,
|
||||
``thumb_id``,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.Item>`.
|
||||
|
||||
You can override this method to customize the thumbnail download path of each image.
|
||||
You can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`thumb_path` method returns
|
||||
``thumbs/<size name>/<request URL hash>.<extension>``.
|
||||
|
||||
|
||||
.. method:: ImagesPipeline.get_media_requests(item, info)
|
||||
|
||||
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
|
||||
|
|
|
|||
|
|
@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
# Your second spider definition
|
||||
...
|
||||
|
||||
configure_logging()
|
||||
settings = get_project_settings()
|
||||
configure_logging(settings)
|
||||
runner = CrawlerRunner(settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
crawl()
|
||||
reactor.run() # the script will block here until the last crawl call is finished
|
||||
|
||||
Different spiders can set different values for the same setting, but when they
|
||||
run in the same process it may be impossible, by design or because of some
|
||||
limitations, to use these different values. What happens in practice is
|
||||
different for different settings:
|
||||
|
||||
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
|
||||
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
|
||||
default one) cannot be read from the per-spider settings. These are applied
|
||||
when the :class:`~scrapy.crawler.CrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerProcess` object is created.
|
||||
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
|
||||
available value is used, and if a spider requests a different reactor an
|
||||
exception will be raised. These are applied when the reactor is installed.
|
||||
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
|
||||
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
|
||||
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
|
||||
the first available value is used. These are applied when the reactor is
|
||||
started.
|
||||
|
||||
.. seealso:: :ref:`run-from-script`.
|
||||
|
||||
.. _distributed-crawls:
|
||||
|
|
@ -243,7 +262,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
|
||||
cookies to spot bot behaviour
|
||||
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
|
||||
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
|
||||
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
||||
directly
|
||||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
|
|
@ -258,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
|
|
|||
|
|
@ -110,6 +110,10 @@ Request objects
|
|||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
|
||||
are casted to :class:`str`.
|
||||
|
||||
:type cookies: dict or list
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
@ -300,7 +304,7 @@ errors if needed::
|
|||
"http://www.httpbin.org/status/404", # Not found error
|
||||
"http://www.httpbin.org/status/500", # server issue
|
||||
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
|
||||
"http://www.httphttpbinbin.org/", # DNS error expected
|
||||
"https://example.invalid/", # DNS error expected
|
||||
]
|
||||
|
||||
def start_requests(self):
|
||||
|
|
@ -335,6 +339,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
|
|
@ -360,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
|
|||
main_url=failure.request.cb_kwargs['main_url'],
|
||||
)
|
||||
|
||||
|
||||
.. _request-fingerprints:
|
||||
|
||||
Request fingerprints
|
||||
--------------------
|
||||
|
||||
There are some aspects of scraping, such as filtering out duplicate requests
|
||||
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
|
||||
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
|
||||
unique identifier from a :class:`~scrapy.http.Request` object: a request
|
||||
fingerprint.
|
||||
|
||||
You often do not need to worry about request fingerprints, the default request
|
||||
fingerprinter works for most projects.
|
||||
|
||||
However, there is no universal way to generate a unique identifier from a
|
||||
request, because different situations require comparing requests differently.
|
||||
For example, sometimes you may need to compare URLs case-insensitively, include
|
||||
URL fragments, exclude certain URL query parameters, include some or all
|
||||
headers, etc.
|
||||
|
||||
To change how request fingerprints are built for your requests, use the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_CLASS
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: :class:`scrapy.utils.request.RequestFingerprinter`
|
||||
|
||||
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
|
||||
import path.
|
||||
|
||||
.. autoclass:: scrapy.utils.request.RequestFingerprinter
|
||||
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: ``'PREVIOUS_VERSION'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'PREVIOUS_VERSION'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy PREVIOUS_VERSION and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'VERSION'``
|
||||
|
||||
This implementation was introduced in Scrapy VERSION to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidade the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'PREVIOUS_VERSION'``).
|
||||
|
||||
|
||||
.. _PREVIOUS_VERSION-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A request fingerprinter is a class that must implement the following method:
|
||||
|
||||
.. method:: fingerprint(self, request)
|
||||
|
||||
Return a :class:`bytes` object that uniquely identifies *request*.
|
||||
|
||||
See also :ref:`request-fingerprint-restrictions`.
|
||||
|
||||
:param request: request to fingerprint
|
||||
:type request: scrapy.http.Request
|
||||
|
||||
Additionally, it may also implement the following methods:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this class method is called to create a request fingerprinter
|
||||
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
|
||||
new instance of the request fingerprinter.
|
||||
|
||||
*crawler* provides access to all Scrapy core components like settings and
|
||||
signals; it is a way for the request fingerprinter to access them and hook
|
||||
its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this request fingerprinter
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
.. classmethod:: from_settings(cls, settings)
|
||||
|
||||
If present, and ``from_crawler`` is not defined, this class method is called
|
||||
to create a request fingerprinter instance from a
|
||||
:class:`~scrapy.settings.Settings` object. It must return a new instance of
|
||||
the request fingerprinter.
|
||||
|
||||
The ``fingerprint`` method of the default request fingerprinter,
|
||||
:class:`scrapy.utils.request.RequestFingerprinter`, uses
|
||||
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
|
||||
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
|
||||
in your ``fingerprint`` method implementation:
|
||||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
For example, to take the value of a request header named ``X-ID`` into
|
||||
account::
|
||||
|
||||
# my_project/settings.py
|
||||
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
|
||||
|
||||
# my_project/utils.py
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
You can also write your own fingerprinting logic from scratch.
|
||||
|
||||
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
|
||||
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
|
||||
|
||||
- Caching saves CPU by ensuring that fingerprints are calculated only once
|
||||
per request, and not once per Scrapy component that needs the fingerprint
|
||||
of a request.
|
||||
|
||||
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
|
||||
request objects do not stay in memory forever just because you have
|
||||
references to them in your cache dictionary.
|
||||
|
||||
For example, to take into account only the URL of a request, without any prior
|
||||
URL canonicalization or taking the request method or body into account::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
If you need to be able to override the request fingerprinting for arbitrary
|
||||
requests from your spider callbacks, you may implement a request fingerprinter
|
||||
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
|
||||
when available, and then falls back to
|
||||
:func:`~scrapy.utils.request.fingerprint`. For example::
|
||||
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
|
||||
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
|
||||
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
|
||||
request fingerprinter::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
|
||||
.. _request-fingerprint-restrictions:
|
||||
|
||||
Request fingerprint restrictions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapy components that use request fingerprints may impose additional
|
||||
restrictions on the format of the fingerprints that your :ref:`request
|
||||
fingerprinter <custom-request-fingerprinter>` generates.
|
||||
|
||||
The following built-in Scrapy components have such restrictions:
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
|
||||
value of :setting:`HTTPCACHE_STORAGE`)
|
||||
|
||||
Request fingerprints must be at least 1 byte long.
|
||||
|
||||
Path and filename length limits of the file system of
|
||||
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
|
||||
the following directory structure is created:
|
||||
|
||||
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
|
||||
|
||||
- first byte of a request fingerprint as hexadecimal
|
||||
|
||||
- fingerprint as hexadecimal
|
||||
|
||||
- filenames up to 16 characters long
|
||||
|
||||
For example, if a request fingerprint is made of 20 bytes (default),
|
||||
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
|
||||
and the name of your spider is ``'my_spider'`` your file system must
|
||||
support a file path like::
|
||||
|
||||
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
|
||||
|
||||
The underlying DBM implementation must support keys as long as twice
|
||||
the number of bytes of a request fingerprint, plus 5. For example,
|
||||
if a request fingerprint is made of 20 bytes (default),
|
||||
45-character-long keys must be supported.
|
||||
|
||||
|
||||
.. _topics-request-meta:
|
||||
|
||||
Request.meta special keys
|
||||
|
|
@ -950,6 +1222,14 @@ TextResponse objects
|
|||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
.. method:: TextResponse.urljoin(url)
|
||||
|
||||
Constructs an absolute url by combining the Response's base url with
|
||||
a possible relative url. The base url shall be extracted from the
|
||||
``<base>`` tag, or just the Response's :attr:`url` if there is no such
|
||||
tag.
|
||||
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
--------------------
|
||||
|
|
|
|||
|
|
@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
|
|||
|
||||
The class used to detect and filter duplicate requests.
|
||||
|
||||
The default (``RFPDupeFilter``) filters based on request fingerprint using
|
||||
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
|
||||
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
|
||||
override its ``request_fingerprint`` method. This method should accept
|
||||
scrapy :class:`~scrapy.Request` object and return its fingerprint
|
||||
(a string).
|
||||
The default (``RFPDupeFilter``) filters based on the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
You can disable filtering of duplicate requests by setting
|
||||
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
|
||||
|
|
@ -1026,6 +1022,16 @@ Default: ``None``
|
|||
|
||||
File name to use for logging output. If ``None``, standard error will be used.
|
||||
|
||||
.. setting:: LOG_FILE_APPEND
|
||||
|
||||
LOG_FILE_APPEND
|
||||
---------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
If ``False``, the log file specified with :setting:`LOG_FILE` will be
|
||||
overwritten (discarding the output from previous runs, if any).
|
||||
|
||||
.. setting:: LOG_FORMAT
|
||||
|
||||
LOG_FORMAT
|
||||
|
|
@ -1566,7 +1572,7 @@ If a reactor is already installed,
|
|||
|
||||
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
|
||||
:exc:`Exception` if the installed reactor does not match the
|
||||
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
|
||||
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
|
||||
:mod:`~twisted.internet.reactor` imports in project files and imported
|
||||
third-party libraries will make Scrapy raise :exc:`Exception` when
|
||||
it checks which reactor is installed.
|
||||
|
|
@ -1587,7 +1593,7 @@ In order to use the reactor installed by Scrapy::
|
|||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1615,7 +1621,7 @@ which raises :exc:`Exception`, becomes::
|
|||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1628,10 +1634,9 @@ which raises :exc:`Exception`, becomes::
|
|||
|
||||
|
||||
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
|
||||
means that Scrapy will not attempt to install any specific reactor, and the
|
||||
default reactor defined by Twisted for the current platform will be used. This
|
||||
is to maintain backward compatibility and avoid possible problems caused by
|
||||
using a non-default reactor.
|
||||
means that Scrapy will install the default reactor defined by Twisted for the
|
||||
current platform. This is to maintain backward compatibility and avoid possible
|
||||
problems caused by using a non-default reactor.
|
||||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
|
|
@ -1645,8 +1650,19 @@ Default: ``2083``
|
|||
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs. For more information about
|
||||
the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
This setting can act as a stopping condition in case of URLs of ever-increasing
|
||||
length, which may be caused for example by a programming error either in the
|
||||
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
|
||||
:setting:`DEPTH_LIMIT`.
|
||||
|
||||
Use ``0`` to allow URLs of any length.
|
||||
|
||||
The default value is copied from the `Microsoft Internet Explorer maximum URL
|
||||
length`_, even though this setting exists for different reasons.
|
||||
|
||||
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
|
||||
.. setting:: USER_AGENT
|
||||
|
||||
|
|
@ -1658,7 +1674,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
|
|||
The default User-Agent to use when crawling, unless overridden. This user agent is
|
||||
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
|
||||
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
|
||||
there is no overridding User-Agent header specified for the request.
|
||||
there is no overriding User-Agent header specified for the request.
|
||||
|
||||
|
||||
Settings documented elsewhere:
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ Available Shortcuts
|
|||
shortcuts
|
||||
|
||||
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
|
||||
and update all related objects accordingly. You can optionaly ask for HTTP
|
||||
and update all related objects accordingly. You can optionally ask for HTTP
|
||||
3xx redirections to not be followed by passing ``redirect=False``
|
||||
|
||||
- ``fetch(request)`` - fetch a new response from the given request and update
|
||||
|
|
|
|||
|
|
@ -51,16 +51,16 @@ Deferred signal handlers
|
|||
========================
|
||||
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, allowing you to run asynchronous code that
|
||||
does not block Scrapy. If a signal handler returns a
|
||||
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
|
||||
:class:`~twisted.internet.defer.Deferred` to fire.
|
||||
or :term:`awaitable objects <awaitable>` from their handlers, allowing
|
||||
you to run asynchronous code that does not block Scrapy. If a signal
|
||||
handler returns one of these objects, Scrapy waits for that asynchronous
|
||||
operation to finish.
|
||||
|
||||
Let's take an example::
|
||||
Let's take an example using :ref:`coroutines <topics-coroutines>`::
|
||||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
start_urls = ['https://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
|
|
@ -68,17 +68,15 @@ Let's take an example::
|
|||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
def item_scraped(self, item):
|
||||
async def item_scraped(self, item):
|
||||
# Send the scraped item to the server
|
||||
d = treq.post(
|
||||
response = await treq.post(
|
||||
'http://example.com/post',
|
||||
json.dumps(item).encode('ascii'),
|
||||
headers={b'Content-Type': [b'application/json']}
|
||||
)
|
||||
|
||||
# The next item will be scraped only after
|
||||
# deferred (d) is fired
|
||||
return d
|
||||
return response
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
|
|
@ -89,7 +87,7 @@ Let's take an example::
|
|||
}
|
||||
|
||||
See the :ref:`topics-signals-ref` below to know which signals support
|
||||
:class:`~twisted.internet.defer.Deferred`.
|
||||
:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -413,7 +411,7 @@ headers_received
|
|||
.. versionadded:: 2.5
|
||||
|
||||
.. signal:: headers_received
|
||||
.. function:: headers_received(headers, request, spider)
|
||||
.. function:: headers_received(headers, body_length, request, spider)
|
||||
|
||||
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||
available for a given request, before downloading any additional content.
|
||||
|
|
|
|||
|
|
@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
method (from a previous spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
|
||||
objects.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
executing any other :meth:`process_spider_exception` in the following
|
||||
|
|
@ -440,4 +440,3 @@ UrlLengthMiddleware
|
|||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are
|
|||
different kinds of default spiders bundled into Scrapy for different purposes.
|
||||
We will talk about those types here.
|
||||
|
||||
.. module:: scrapy.spiders
|
||||
:synopsis: Spiders base class, spider manager and spider middleware
|
||||
|
||||
.. _topics-spiders-ref:
|
||||
|
||||
scrapy.Spider
|
||||
=============
|
||||
|
||||
.. class:: scrapy.spiders.Spider()
|
||||
.. class:: scrapy.spiders.Spider
|
||||
.. class:: scrapy.Spider()
|
||||
|
||||
This is the simplest spider, and the one from which every other spider
|
||||
|
|
@ -122,6 +119,11 @@ scrapy.Spider
|
|||
send log messages through it as described on
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. attribute:: state
|
||||
|
||||
A dict you can use to persist some spider state between batches.
|
||||
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
|
||||
|
||||
.. method:: from_crawler(crawler, *args, **kwargs)
|
||||
|
||||
This is the class method used by Scrapy to create your spiders.
|
||||
|
|
@ -367,7 +369,7 @@ CrawlSpider
|
|||
described below. If multiple rules match the same link, the first one
|
||||
will be used, according to the order they're defined in this attribute.
|
||||
|
||||
This spider also exposes an overrideable method:
|
||||
This spider also exposes an overridable method:
|
||||
|
||||
.. method:: parse_start_url(response, **kwargs)
|
||||
|
||||
|
|
@ -529,7 +531,7 @@ XMLFeedSpider
|
|||
itertag = 'n:url'
|
||||
# ...
|
||||
|
||||
Apart from these new attributes, this spider has the following overrideable
|
||||
Apart from these new attributes, this spider has the following overridable
|
||||
methods too:
|
||||
|
||||
.. method:: adapt_response(response)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
|
|||
large changes.
|
||||
* *B* is the release number. This will include many changes including features
|
||||
and things that possibly break backward compatibility, although we strive to
|
||||
keep theses cases at a minimum.
|
||||
keep these cases at a minimum.
|
||||
* *C* is the bugfix release number.
|
||||
|
||||
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
A spider that generate light requests to meassure QPS troughput
|
||||
A spider that generate light requests to meassure QPS throughput
|
||||
|
||||
usage:
|
||||
|
||||
|
|
|
|||
1
pylintrc
1
pylintrc
|
|
@ -112,6 +112,7 @@ disable=abstract-method,
|
|||
unused-private-member,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
use-implicit-booleaness-not-comparison,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
|
|
|
|||
|
|
@ -20,5 +20,4 @@ addopts =
|
|||
--ignore=docs/utils
|
||||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
filterwarnings=
|
||||
ignore::DeprecationWarning:twisted.web.test.test_webclient
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.5.0
|
||||
2.6.1
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
|||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 6):
|
||||
print("Scrapy %s requires Python 3.6+" % __version__)
|
||||
if sys.version_info < (3, 7):
|
||||
print(f"Scrapy {__version__} requires Python 3.7+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import argparse
|
||||
import cProfile
|
||||
import inspect
|
||||
import pkg_resources
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
|
|
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
|
||||
parser.description = cmd.long_desc()
|
||||
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler='resolve',
|
||||
description=cmd.long_desc())
|
||||
settings.setdict(cmd.default_settings, priority='command')
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
cmd.crawler_process = CrawlerProcess(settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from optparse import OptionGroup
|
||||
import argparse
|
||||
from typing import Any, Dict
|
||||
|
||||
from twisted.python import failure
|
||||
|
|
@ -43,14 +43,14 @@ class ScrapyCommand:
|
|||
|
||||
def long_desc(self):
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since no post-formatting will
|
||||
"help" command. It can contain newlines since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
|
@ -59,22 +59,20 @@ class ScrapyCommand:
|
|||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
group = parser.add_argument_group(title='Global Options')
|
||||
group.add_argument("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_argument("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_argument("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_argument("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
try:
|
||||
|
|
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_argument("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_argument("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
|
||||
class ScrapyHelpFormatter(argparse.HelpFormatter):
|
||||
"""
|
||||
Help Formatter for scrapy command line help messages.
|
||||
"""
|
||||
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
|
||||
super().__init__(prog, indent_increment=indent_increment,
|
||||
max_help_position=max_help_position, width=width)
|
||||
|
||||
def _join_parts(self, part_strings):
|
||||
parts = self.format_part_strings(part_strings)
|
||||
return super()._join_parts(parts)
|
||||
|
||||
def format_part_strings(self, part_strings):
|
||||
"""
|
||||
Underline and title case command line help message headers.
|
||||
"""
|
||||
if part_strings and part_strings[0].startswith("usage: "):
|
||||
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
|
||||
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
|
||||
for index in headings[::-1]:
|
||||
char = '-' if "Global Options" in part_strings[index] else '='
|
||||
part_strings[index] = part_strings[index][:-2].title()
|
||||
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
|
||||
part_strings.insert(index + 1, underline)
|
||||
return part_strings
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
parser.add_argument("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -44,16 +44,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_argument("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_argument("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
|
|||
|
|
@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
parser.add_argument("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_argument("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_argument("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_argument("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_argument("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def _start_requests(spider):
|
||||
yield self.prepare_request(spider, Request(url), opts)
|
||||
self.spidercls.start_requests = _start_requests
|
||||
if self.spidercls:
|
||||
self.spidercls.start_requests = _start_requests
|
||||
|
||||
def start_parsing(self, url, opts):
|
||||
self.crawler_process.crawl(self.spidercls, **opts.spargs)
|
||||
|
|
|
|||
|
|
@ -18,16 +18,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
parser.add_argument("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_argument("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_argument("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _start_crawler_thread(self):
|
||||
t = Thread(target=self.crawler_process.start,
|
||||
kwargs={'stop_after_crawl': False})
|
||||
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ class Command(fetch.Command):
|
|||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
parser.add_argument('--headers', help=argparse.SUPPRESS)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
|
|||
settings=settings,
|
||||
crawler=crawler,
|
||||
)
|
||||
msg = """
|
||||
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
|
||||
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
|
||||
Please upgrade your context factory class to handle them or ignore them.""" % (
|
||||
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
|
||||
msg = (
|
||||
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
|
||||
"a `method` argument (type OpenSSL.SSL method, e.g. "
|
||||
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
|
||||
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
|
||||
"context factory class to handle them or ignore them."
|
||||
)
|
||||
warnings.warn(msg)
|
||||
|
||||
return context_factory
|
||||
|
|
|
|||
|
|
@ -102,11 +102,11 @@ class FTPDownloadHandler:
|
|||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
respcls = responsetypes.from_args(url=request.url)
|
||||
protocol.close()
|
||||
body = protocol.filename or protocol.body.read()
|
||||
headers = {"local filename": protocol.filename or '', "size": protocol.size}
|
||||
return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
|
||||
body = to_bytes(protocol.filename or protocol.body.read())
|
||||
respcls = responsetypes.from_args(url=request.url, body=body)
|
||||
return respcls(url=request.url, status=200, body=body, headers=headers)
|
||||
|
||||
def _failed(self, result, request):
|
||||
message = result.getErrorMessage()
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ class TunnelingAgent(Agent):
|
|||
# proxy host and port are required for HTTP pool `key`
|
||||
# otherwise, same remote host connection request could reuse
|
||||
# a cached tunneled connection to a different proxy
|
||||
key = key + self._proxyConf
|
||||
key += self._proxyConf
|
||||
return super()._requestWithEndpoint(
|
||||
key=key,
|
||||
endpoint=endpoint,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
from typing import Callable, Union
|
||||
from typing import Callable, Union, cast
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
@defer.inlineCallbacks
|
||||
def process_request(request: Request):
|
||||
for method in self.methods['process_request']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
def process_exception(failure: Failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
|
|||
|
|
@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
verifyHostname(connection, self._hostnameASCII)
|
||||
except (CertificateError, VerificationError) as e:
|
||||
logger.warning(
|
||||
'Remote certificate is not valid for hostname "{}"; {}'.format(
|
||||
self._hostnameASCII, e))
|
||||
'Remote certificate is not valid for hostname "%s"; %s',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
'Ignoring error while verifying certificate '
|
||||
'from host "{}" (exception: {})'.format(
|
||||
self._hostnameASCII, repr(e)))
|
||||
'from host "%s" (exception: %r)',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
|
||||
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from time import time
|
|||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
|
||||
from scrapy.http import Headers
|
||||
|
|
@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
request.meta['download_latency'] = self.headers_time - self.start_time
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
|
|
@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
from twisted.internet import reactor
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
|
|
|||
|
|
@ -285,8 +285,8 @@ class Stream:
|
|||
|
||||
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
|
||||
|
||||
bytes_to_send_size = bytes_to_send_size - chunk_size
|
||||
self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
|
||||
bytes_to_send_size -= chunk_size
|
||||
self.metadata['remaining_content_length'] -= chunk_size
|
||||
|
||||
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
|
||||
|
||||
|
|
|
|||
|
|
@ -156,7 +156,7 @@ class Scraper:
|
|||
callback = result.request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
dfd = defer_succeed(result)
|
||||
dfd.addCallback(callback, **result.request.cb_kwargs)
|
||||
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
|
||||
else: # result is a Failure
|
||||
result.request = request
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Spider Middleware manager
|
|||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
from itertools import islice
|
||||
from typing import Any, Callable, Generator, Iterable, Union
|
||||
from typing import Any, Callable, Generator, Iterable, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
|
||||
spider: Spider) -> Any:
|
||||
for method in self.methods['process_spider_input']:
|
||||
method = cast(Callable, method)
|
||||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.utils.log import (
|
|||
configure_logging,
|
||||
get_scrapy_root_handler,
|
||||
install_scrapy_root_handler,
|
||||
log_reactor_info,
|
||||
log_scrapy_info,
|
||||
LogCounterHandler,
|
||||
)
|
||||
|
|
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class Crawler:
|
||||
|
||||
def __init__(self, spidercls, settings=None):
|
||||
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
|
||||
if isinstance(spidercls, Spider):
|
||||
raise ValueError('The spidercls argument must be a class, not an object')
|
||||
|
||||
|
|
@ -50,6 +51,7 @@ class Crawler:
|
|||
self.spidercls.update_settings(self.settings)
|
||||
|
||||
self.signals = SignalManager(self)
|
||||
|
||||
self.stats = load_object(self.settings['STATS_CLASS'])(self)
|
||||
|
||||
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
|
||||
|
|
@ -69,6 +71,26 @@ class Crawler:
|
|||
|
||||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
self.request_fingerprinter = create_instance(
|
||||
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
|
||||
settings=self.settings,
|
||||
crawler=self,
|
||||
)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
# but before something imports twisted.internet.reactor
|
||||
if reactor_class:
|
||||
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
else:
|
||||
from twisted.internet import default
|
||||
default.install()
|
||||
log_reactor_info()
|
||||
if reactor_class:
|
||||
verify_installed_reactor(reactor_class)
|
||||
|
||||
self.extensions = ExtensionManager.from_crawler(self)
|
||||
|
||||
self.settings.freeze()
|
||||
|
|
@ -153,7 +175,6 @@ class CrawlerRunner:
|
|||
self._crawlers = set()
|
||||
self._active = set()
|
||||
self.bootstrap_failed = False
|
||||
self._handle_twisted_reactor()
|
||||
|
||||
@property
|
||||
def spiders(self):
|
||||
|
|
@ -247,10 +268,6 @@ class CrawlerRunner:
|
|||
while self._active:
|
||||
yield defer.DeferredList(self._active)
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
|
||||
|
||||
|
||||
class CrawlerProcess(CrawlerRunner):
|
||||
"""
|
||||
|
|
@ -278,7 +295,6 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
||||
|
|
@ -298,7 +314,12 @@ class CrawlerProcess(CrawlerRunner):
|
|||
{'signame': signame})
|
||||
reactor.callFromThread(self._stop_reactor)
|
||||
|
||||
def start(self, stop_after_crawl=True):
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings, init_reactor=True)
|
||||
|
||||
def start(self, stop_after_crawl=True, install_signal_handlers=True):
|
||||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
|
|
@ -309,6 +330,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
||||
:param bool install_signal_handlers: whether to install the shutdown
|
||||
handlers (default: True)
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
if stop_after_crawl:
|
||||
|
|
@ -318,6 +342,8 @@ class CrawlerProcess(CrawlerRunner):
|
|||
return
|
||||
d.addBoth(self._stop_reactor)
|
||||
|
||||
if install_signal_handlers:
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
resolver_class = load_object(self.settings["DNS_RESOLVER"])
|
||||
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
|
||||
resolver.install_on_reactor()
|
||||
|
|
@ -337,8 +363,3 @@ class CrawlerProcess(CrawlerRunner):
|
|||
reactor.stop()
|
||||
except RuntimeError: # raised if already stopped or in shutdown stage
|
||||
pass
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from tldextract import TLDExtract
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_split_domain = TLDExtract(include_psl_private_domains=True)
|
||||
|
||||
|
||||
def _is_public_domain(domain):
|
||||
parts = _split_domain(domain)
|
||||
return not parts.domain
|
||||
|
||||
|
||||
class CookiesMiddleware:
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
|
||||
|
|
@ -23,14 +34,29 @@ class CookiesMiddleware:
|
|||
raise NotConfigured
|
||||
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
|
||||
|
||||
def _process_cookies(self, cookies, *, jar, request):
|
||||
for cookie in cookies:
|
||||
cookie_domain = cookie.domain
|
||||
if cookie_domain.startswith('.'):
|
||||
cookie_domain = cookie_domain[1:]
|
||||
|
||||
request_domain = urlparse_cached(request).hostname.lower()
|
||||
|
||||
if cookie_domain and _is_public_domain(cookie_domain):
|
||||
if cookie_domain != request_domain:
|
||||
continue
|
||||
cookie.domain = request_domain
|
||||
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if request.meta.get('dont_merge_cookies', False):
|
||||
return
|
||||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
for cookie in self._get_request_cookies(jar, request):
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
# set Cookie header
|
||||
request.headers.pop('Cookie', None)
|
||||
|
|
@ -44,7 +70,9 @@ class CookiesMiddleware:
|
|||
# extract cookies from Set-Cookie and drop invalid/expired cookies
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
jar.extract_cookies(response, request)
|
||||
cookies = jar.make_cookies(response, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
self._debug_set_cookie(response, spider)
|
||||
|
||||
return response
|
||||
|
|
@ -80,8 +108,8 @@ class CookiesMiddleware:
|
|||
logger.warning(msg.format(request, cookie, key))
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], str):
|
||||
decoded[key] = cookie[key]
|
||||
if isinstance(cookie[key], (bool, float, int, str)):
|
||||
decoded[key] = str(cookie[key])
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.url import url_is_from_any_domain
|
||||
|
||||
|
||||
class HttpAuthMiddleware:
|
||||
|
|
@ -24,8 +28,23 @@ class HttpAuthMiddleware:
|
|||
pwd = getattr(spider, 'http_pass', '')
|
||||
if usr or pwd:
|
||||
self.auth = basic_auth_header(usr, pwd)
|
||||
if not hasattr(spider, 'http_auth_domain'):
|
||||
warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
|
||||
'problems if the spider makes requests to several different domains. http_auth_domain '
|
||||
'will be set to the domain of the first request, please set it to the correct value '
|
||||
'explicitly.',
|
||||
category=ScrapyDeprecationWarning)
|
||||
self.domain_unset = True
|
||||
else:
|
||||
self.domain = spider.http_auth_domain
|
||||
self.domain_unset = False
|
||||
|
||||
def process_request(self, request, spider):
|
||||
auth = getattr(self, 'auth', None)
|
||||
if auth and b'Authorization' not in request.headers:
|
||||
request.headers[b'Authorization'] = auth
|
||||
domain = urlparse_cached(request).hostname
|
||||
if self.domain_unset:
|
||||
self.domain = domain
|
||||
self.domain_unset = False
|
||||
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
|
||||
request.headers[b'Authorization'] = auth
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class HttpCacheMiddleware:
|
|||
self.stats.inc_value('httpcache/miss', spider=spider)
|
||||
if self.ignore_missing:
|
||||
self.stats.inc_value('httpcache/ignore', spider=spider)
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
raise IgnoreRequest(f"Ignored request not in cache: {request}")
|
||||
return None # first time request
|
||||
|
||||
# Return cached response only if not expired
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse
|
|||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
|
||||
|
|
@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_redirect_request(source_request, *, url, **kwargs):
|
||||
redirect_request = source_request.replace(
|
||||
url=url,
|
||||
**kwargs,
|
||||
cookies=None,
|
||||
)
|
||||
if 'Cookie' in redirect_request.headers:
|
||||
source_request_netloc = urlparse_cached(source_request).netloc
|
||||
redirect_request_netloc = urlparse_cached(redirect_request).netloc
|
||||
if source_request_netloc != redirect_request_netloc:
|
||||
del redirect_request.headers['Cookie']
|
||||
return redirect_request
|
||||
|
||||
|
||||
class BaseRedirectMiddleware:
|
||||
|
||||
enabled_setting = 'REDIRECT_ENABLED'
|
||||
|
|
@ -47,10 +62,15 @@ class BaseRedirectMiddleware:
|
|||
raise IgnoreRequest("max redirections reached")
|
||||
|
||||
def _redirect_request_using_get(self, request, redirect_url):
|
||||
redirected = request.replace(url=redirect_url, method='GET', body='')
|
||||
redirected.headers.pop('Content-Type', None)
|
||||
redirected.headers.pop('Content-Length', None)
|
||||
return redirected
|
||||
redirect_request = _build_redirect_request(
|
||||
request,
|
||||
url=redirect_url,
|
||||
method='GET',
|
||||
body='',
|
||||
)
|
||||
redirect_request.headers.pop('Content-Type', None)
|
||||
redirect_request.headers.pop('Content-Length', None)
|
||||
return redirect_request
|
||||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
|
|
@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
if response.status in (301, 307, 308) or request.method == 'HEAD':
|
||||
redirected = request.replace(url=redirected_url)
|
||||
redirected = _build_redirect_request(request, url=redirected_url)
|
||||
return self._redirect(redirected, request, spider, response.status)
|
||||
|
||||
redirected = self._redirect_request_using_get(request, redirected_url)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
An extension to retry failed requests that are potentially caused by temporary
|
||||
problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
You can change the behaviour of this middleware by modifing the scraping settings:
|
||||
You can change the behaviour of this middleware by modifying the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,22 @@
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.python import global_object_name, to_bytes
|
||||
from scrapy.utils.request import request_httprepr
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
from twisted.web import http
|
||||
|
||||
|
||||
def get_header_size(headers):
|
||||
size = 0
|
||||
for key, value in headers.items():
|
||||
if isinstance(value, (list, tuple)):
|
||||
for v in value:
|
||||
size += len(b": ") + len(key) + len(v)
|
||||
return size + len(b'\r\n') * (len(headers.keys()) - 1)
|
||||
|
||||
|
||||
def get_status_size(response_status):
|
||||
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
|
||||
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
|
||||
|
||||
|
||||
class DownloaderStats:
|
||||
|
|
@ -24,7 +39,8 @@ class DownloaderStats:
|
|||
def process_response(self, request, response, spider):
|
||||
self.stats.inc_value('downloader/response_count', spider=spider)
|
||||
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
|
||||
reslen = len(response_httprepr(response))
|
||||
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
|
||||
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
|
||||
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import logging
|
||||
import os
|
||||
from typing import Optional, Set, Type, TypeVar
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
from scrapy.utils.request import referer_str, RequestFingerprinter
|
||||
|
||||
|
||||
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
|
||||
|
|
@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
|
|||
class RFPDupeFilter(BaseDupeFilter):
|
||||
"""Request Fingerprint duplicates filter"""
|
||||
|
||||
def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[str] = None,
|
||||
debug: bool = False,
|
||||
*,
|
||||
fingerprinter=None,
|
||||
) -> None:
|
||||
self.file = None
|
||||
self.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
self.fingerprints: Set[str] = set()
|
||||
self.logdupes = True
|
||||
self.debug = debug
|
||||
|
|
@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
self.fingerprints.update(x.rstrip() for x in self.file)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
|
||||
debug = settings.getbool('DUPEFILTER_DEBUG')
|
||||
return cls(job_dir(settings), debug)
|
||||
try:
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their '__init__' "
|
||||
"method to support a 'fingerprinter' parameter or reimplement "
|
||||
"the 'from_settings' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls(job_dir(settings), debug)
|
||||
result.fingerprinter = fingerprinter
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
try:
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
fingerprinter=crawler.request_fingerprinter,
|
||||
)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their overridden "
|
||||
"'__init__' method and 'from_settings' class method to "
|
||||
"support a 'fingerprinter' parameter, or reimplement the "
|
||||
"'from_crawler' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls.from_settings(crawler.settings)
|
||||
result.fingerprinter = crawler.request_fingerprinter
|
||||
return result
|
||||
|
||||
def request_seen(self, request: Request) -> bool:
|
||||
fp = self.request_fingerprint(request)
|
||||
|
|
@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
return False
|
||||
|
||||
def request_fingerprint(self, request: Request) -> str:
|
||||
return request_fingerprint(request)
|
||||
return self.fingerprinter.fingerprint(request).hex()
|
||||
|
||||
def close(self, reason: str) -> None:
|
||||
if self.file:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ class BaseItemExporter:
|
|||
self._configure(kwargs, dont_fail=dont_fail)
|
||||
|
||||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
"""Configure the exporter by popping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||
"""
|
||||
|
|
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
return serializer(value)
|
||||
|
||||
def _serialize_value(self, value):
|
||||
if isinstance(value, _BaseItem):
|
||||
if isinstance(value, Item):
|
||||
return self.export_item(value)
|
||||
elif is_item(value):
|
||||
return dict(self._serialize_item(value))
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ import sys
|
|||
import warnings
|
||||
from datetime import datetime
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import implementer, Interface
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
|
|
@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
|||
kwargs['feed_options'] = feed_options
|
||||
else:
|
||||
warnings.warn(
|
||||
"{} does not support the 'feed_options' keyword argument. Add a "
|
||||
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove this "
|
||||
"warning. This parameter will become mandatory in a future "
|
||||
"version of Scrapy."
|
||||
.format(builder.__qualname__),
|
||||
"version of Scrapy.",
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return builder(*preargs, uri, *args, **kwargs)
|
||||
|
|
@ -356,32 +355,28 @@ class FeedExporter:
|
|||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
|
||||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
|
||||
)
|
||||
d.addErrback(
|
||||
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
self._handle_store_error, logmsg, spider, type(slot.storage).__name__
|
||||
)
|
||||
return d
|
||||
|
||||
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
|
||||
def _handle_store_error(self, f, logmsg, spider, slot_type):
|
||||
logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
"Error storing %s", logmsg,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
|
||||
|
||||
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
|
||||
def _handle_store_success(self, f, logmsg, spider, slot_type):
|
||||
logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
"Stored %s", logmsg,
|
||||
extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
|
||||
|
||||
|
|
@ -474,10 +469,10 @@ class FeedExporter:
|
|||
for uri_template, values in self.feeds.items():
|
||||
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
|
||||
logger.error(
|
||||
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
|
||||
''.format(uri_template)
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
|
||||
uri_template
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
|
@ -526,10 +521,15 @@ class FeedExporter:
|
|||
instance = build_instance(feedcls)
|
||||
method_name = '__new__'
|
||||
if instance is None:
|
||||
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
|
||||
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
def _get_uri_params(
|
||||
self,
|
||||
spider: Spider,
|
||||
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
|
||||
slot: Optional[_FeedSlot] = None,
|
||||
) -> dict:
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
|
|
@ -537,9 +537,18 @@ class FeedExporter:
|
|||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
original_params = params.copy()
|
||||
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
|
||||
new_params = uripar_function(params, spider)
|
||||
if new_params is None or original_params != params:
|
||||
warnings.warn(
|
||||
'Modifying the params dictionary in-place in the function defined in '
|
||||
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
|
||||
'setting is deprecated. The function must return a new dictionary '
|
||||
'instead.',
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return new_params if new_params is not None else params
|
||||
|
||||
def _load_filter(self, feed_options):
|
||||
# load the item filter if declared else load the default filter class
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -226,7 +225,9 @@ class DbmCacheStorage:
|
|||
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
|
||||
self.db = self.dbmodule.open(dbpath, 'c')
|
||||
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.db.close()
|
||||
|
|
@ -239,12 +240,12 @@ class DbmCacheStorage:
|
|||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
|
|
@ -255,7 +256,7 @@ class DbmCacheStorage:
|
|||
self.db[f'{key}_time'] = str(time())
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
db = self.db
|
||||
tkey = f'{key}_time'
|
||||
if tkey not in db:
|
||||
|
|
@ -267,9 +268,6 @@ class DbmCacheStorage:
|
|||
|
||||
return pickle.loads(db[f'{key}_data'])
|
||||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class FilesystemCacheStorage:
|
||||
|
||||
|
|
@ -280,9 +278,11 @@ class FilesystemCacheStorage:
|
|||
self._open = gzip.open if self.use_gzip else open
|
||||
|
||||
def open_spider(self, spider):
|
||||
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
|
||||
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
|
||||
extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
|
|
@ -299,7 +299,7 @@ class FilesystemCacheStorage:
|
|||
url = metadata.get('response_url')
|
||||
status = metadata['status']
|
||||
headers = Headers(headers_raw_to_dict(rawheaders))
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
|
|
@ -329,7 +329,7 @@ class FilesystemCacheStorage:
|
|||
f.write(request.body)
|
||||
|
||||
def _get_request_path(self, spider, request):
|
||||
key = request_fingerprint(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
return os.path.join(self.cachedir, spider.name, key[0:2], key)
|
||||
|
||||
def _read_meta(self, spider, request):
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ class MemoryUsage:
|
|||
self.crawler = crawler
|
||||
self.warned = False
|
||||
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
|
||||
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
|
||||
self.mail = MailSender.from_settings(crawler.settings)
|
||||
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
|
||||
|
|
@ -77,7 +77,7 @@ class MemoryUsage:
|
|||
def _check_limit(self):
|
||||
if self.get_virtual_size() > self.limit:
|
||||
self.crawler.stats.set_value('memusage/limit_reached', 1)
|
||||
mem = self.limit/1024/1024
|
||||
mem = self.limit / 1024 / 1024
|
||||
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
@ -94,11 +94,11 @@ class MemoryUsage:
|
|||
self.crawler.stop()
|
||||
|
||||
def _check_warning(self):
|
||||
if self.warned: # warn only once
|
||||
if self.warned: # warn only once
|
||||
return
|
||||
if self.get_virtual_size() > self.warning:
|
||||
self.crawler.stats.set_value('memusage/warning_reached', 1)
|
||||
mem = self.warning/1024/1024
|
||||
mem = self.warning / 1024 / 1024
|
||||
logger.warning("Memory usage reached %(memusage)dM",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from scrapy import signals
|
|||
from scrapy.mail import MailSender
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
||||
class StatsMailer:
|
||||
|
||||
def __init__(self, stats, recipients, mail):
|
||||
|
|
|
|||
|
|
@ -142,10 +142,6 @@ class WrappedRequest:
|
|||
"""
|
||||
return self.request.meta.get('is_unverifiable', False)
|
||||
|
||||
def get_origin_req_host(self):
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
# python3 uses attributes instead of methods
|
||||
@property
|
||||
def full_url(self):
|
||||
return self.get_full_url()
|
||||
|
|
@ -164,7 +160,7 @@ class WrappedRequest:
|
|||
|
||||
@property
|
||||
def origin_req_host(self):
|
||||
return self.get_origin_req_host()
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
def has_header(self, name):
|
||||
return name in self.request.headers
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
|
|||
"""
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from typing import Generator, Tuple
|
||||
from urllib.parse import urljoin
|
||||
|
|
@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
http_content_type_encoding, resolve_encoding)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
|
@ -66,13 +64,6 @@ class TextResponse(Response):
|
|||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
warnings.warn('Response.body_as_unicode() is deprecated, '
|
||||
'please use Response.text instead.',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.text
|
||||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
|
|
|||
|
|
@ -9,45 +9,15 @@ from collections.abc import MutableMapping
|
|||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from typing import Dict
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
class _BaseItem(object_ref):
|
||||
"""
|
||||
Temporary class used internally to avoid the deprecation
|
||||
warning raised by isinstance checks using BaseItem.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class _BaseItemMeta(ABCMeta):
|
||||
def __instancecheck__(cls, instance):
|
||||
if cls is BaseItem:
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
|
||||
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
||||
"""
|
||||
Deprecated, please use :class:`scrapy.item.Item` instead
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
class ItemMeta(_BaseItemMeta):
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
.. _metaclass: https://realpython.com/python-metaclasses
|
||||
|
|
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
|
|||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
fields: Dict[str, Field] = {}
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
||||
fields: Dict[str, Field]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __iter__(self):
|
||||
return iter(self._values)
|
||||
|
||||
__hash__ = BaseItem.__hash__
|
||||
__hash__ = object_ref.__hash__
|
||||
|
||||
def keys(self):
|
||||
return self._values.keys()
|
||||
|
|
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
|
|||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class LxmlParserLinkExtractor:
|
|||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
||||
The subclass should override it if neccessary
|
||||
The subclass should override it if necessary
|
||||
"""
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader):
|
|||
|
||||
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
|
||||
if selector is None and response is not None:
|
||||
selector = self.default_selector_class(response)
|
||||
try:
|
||||
selector = self.default_selector_class(response)
|
||||
except AttributeError:
|
||||
selector = None
|
||||
context.update(response=response)
|
||||
super().__init__(item=item, selector=selector, parent=parent, **context)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import logging
|
||||
import pprint
|
||||
from collections import defaultdict, deque
|
||||
from typing import Callable, Deque, Dict
|
||||
from typing import Callable, Deque, Dict, Optional, cast, Iterable
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -9,7 +9,7 @@ from scrapy import Spider
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
|
||||
from scrapy.utils.defer import process_parallel, process_chain
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -21,7 +21,8 @@ class MiddlewareManager:
|
|||
|
||||
def __init__(self, *middlewares):
|
||||
self.middlewares = middlewares
|
||||
self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
|
||||
# Optional because process_spider_output and process_spider_exception can be None
|
||||
self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
|
||||
for mw in middlewares:
|
||||
self._add_middleware(mw)
|
||||
|
||||
|
|
@ -64,14 +65,12 @@ class MiddlewareManager:
|
|||
self.methods['close_spider'].appendleft(mw.close_spider)
|
||||
|
||||
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_parallel(self.methods[methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_parallel(methods, obj, *args)
|
||||
|
||||
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain(self.methods[methodname], obj, *args)
|
||||
|
||||
def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain_both(self.methods[cb_methodname],
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_chain(methods, obj, *args)
|
||||
|
||||
def open_spider(self, spider: Spider) -> Deferred:
|
||||
return self._process_parallel('open_spider', spider)
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class S3FilesStore:
|
|||
AWS_USE_SSL = None
|
||||
AWS_VERIFY = None
|
||||
|
||||
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
||||
POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
||||
HEADERS = {
|
||||
'Cache-Control': 'max-age=172800',
|
||||
}
|
||||
|
|
@ -142,7 +142,7 @@ class S3FilesStore:
|
|||
**extra)
|
||||
|
||||
def _headers_to_botocore_kwargs(self, headers):
|
||||
""" Convert headers to botocore keyword agruments.
|
||||
""" Convert headers to botocore keyword arguments.
|
||||
"""
|
||||
# This is required while we need to support both boto and botocore.
|
||||
mapping = CaselessDict({
|
||||
|
|
@ -190,7 +190,7 @@ class GCSFilesStore:
|
|||
CACHE_CONTROL = 'max-age=172800'
|
||||
|
||||
# The bucket's default object ACL will be applied to the object.
|
||||
# Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
||||
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
||||
POLICY = None
|
||||
|
||||
def __init__(self, uri):
|
||||
|
|
@ -222,8 +222,8 @@ class GCSFilesStore:
|
|||
return {'checksum': checksum, 'last_modified': last_modified}
|
||||
else:
|
||||
return {}
|
||||
|
||||
return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
|
||||
blob_path = self._get_blob_path(path)
|
||||
return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
|
||||
|
||||
def _get_content_type(self, headers):
|
||||
if headers and 'Content-Type' in headers:
|
||||
|
|
@ -231,8 +231,12 @@ class GCSFilesStore:
|
|||
else:
|
||||
return 'application/octet-stream'
|
||||
|
||||
def _get_blob_path(self, path):
|
||||
return self.prefix + path
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
blob = self.bucket.blob(self.prefix + path)
|
||||
blob_path = self._get_blob_path(path)
|
||||
blob = self.bucket.blob(blob_path)
|
||||
blob.cache_control = self.CACHE_CONTROL
|
||||
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
|
||||
return threads.deferToThread(
|
||||
|
|
@ -291,7 +295,7 @@ class FilesPipeline(MediaPipeline):
|
|||
"""Abstract pipeline that implement the file downloading
|
||||
|
||||
This pipeline tries to minimize network transfers and file processing,
|
||||
doing stat of the files and determining if file is new, uptodate or
|
||||
doing stat of the files and determining if file is new, up-to-date or
|
||||
expired.
|
||||
|
||||
``new`` files are those that pipeline never processed and needs to be
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
yield path, image, buf
|
||||
|
||||
for thumb_id, size in self.thumbs.items():
|
||||
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
|
||||
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
|
||||
thumb_image, thumb_buf = self.convert_image(image, size)
|
||||
yield thumb_path, thumb_image, thumb_buf
|
||||
|
||||
|
|
@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
|
|||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return f'full/{image_guid}.jpg'
|
||||
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None):
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
|
||||
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from scrapy.settings import Settings
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
||||
|
|
@ -77,6 +76,7 @@ class MediaPipeline:
|
|||
except AttributeError:
|
||||
pipe = cls()
|
||||
pipe.crawler = crawler
|
||||
pipe._fingerprinter = crawler.request_fingerprinter
|
||||
return pipe
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -90,7 +90,7 @@ class MediaPipeline:
|
|||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info, item):
|
||||
fp = request_fingerprint(request)
|
||||
fp = self._fingerprinter.fingerprint(request)
|
||||
cb = request.callback or (lambda _: _)
|
||||
eb = request.errback
|
||||
request.callback = None
|
||||
|
|
@ -121,7 +121,7 @@ class MediaPipeline:
|
|||
def _make_compatible(self):
|
||||
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
|
||||
methods = [
|
||||
"file_path", "media_to_download", "media_downloaded",
|
||||
"file_path", "thumb_path", "media_to_download", "media_downloaded",
|
||||
"file_downloaded", "image_downloaded", "get_images"
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
|
|||
return len(self.attributes)
|
||||
|
||||
def _to_dict(self):
|
||||
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
for k, v in self.items()}
|
||||
|
||||
def _get_key(self, key_value):
|
||||
return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
|
||||
else str(key_value))
|
||||
|
||||
def copy_to_dict(self):
|
||||
"""
|
||||
Make a copy of current settings and convert to a dict.
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
|
|||
FEED_EXPORTERS_BASE = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'jl': 'scrapy.exporters.JsonLinesItemExporter',
|
||||
'csv': 'scrapy.exporters.CsvItemExporter',
|
||||
'xml': 'scrapy.exporters.XmlItemExporter',
|
||||
|
|
@ -207,6 +208,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
|
|||
LOG_STDOUT = False
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
LOG_FILE = None
|
||||
LOG_FILE_APPEND = True
|
||||
LOG_SHORT_NAMES = False
|
||||
|
||||
SCHEDULER_DEBUG = False
|
||||
|
|
@ -245,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
|
|||
REFERER_ENABLED = True
|
||||
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_node(self, response, selector):
|
||||
"""This method must be overriden with your custom spider functionality"""
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
if hasattr(self, 'parse_item'): # backward compatibility
|
||||
return self.parse_item(response, selector)
|
||||
raise NotImplementedError
|
||||
|
|
@ -113,7 +113,7 @@ class CSVFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_row(self, response, row):
|
||||
"""This method must be overriden with your custom spider functionality"""
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
raise NotImplementedError
|
||||
|
||||
def parse_rows(self, response):
|
||||
|
|
@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
|
|||
process_results methods for pre and post-processing purposes.
|
||||
"""
|
||||
|
||||
for row in csviter(response, self.delimiter, self.headers, self.quotechar):
|
||||
for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
|
||||
ret = iterate_spider_output(self.parse_row(response, row))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
|
|
|||
|
|
@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
|
|||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
|
|||
message = (
|
||||
'The -t command line option is deprecated in favor of '
|
||||
'specifying the output format within the output URI. See the '
|
||||
'documentation of the -o and -O options for more information.',
|
||||
'documentation of the -o and -O options for more information.'
|
||||
)
|
||||
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
|
||||
return {output[0]: {'format': output_format}}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
|
|||
@wraps(_embed_ipython_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
config = load_default_config()
|
||||
# Always use .instace() to ensure _instance propagation to all parents
|
||||
# Always use .instance() to ensure _instance propagation to all parents
|
||||
# this is needed for <TAB> completion works well for new imports
|
||||
# and clear the instance to always have the fresh env
|
||||
# on repeated breaks like with inspect_response()
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class CaselessDict(dict):
|
|||
return key.lower()
|
||||
|
||||
def normvalue(self, value):
|
||||
"""Method to normalize values prior to be setted"""
|
||||
"""Method to normalize values prior to be set"""
|
||||
return value
|
||||
|
||||
def get(self, key, def_val=None):
|
||||
|
|
|
|||
|
|
@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds
|
|||
"""
|
||||
import asyncio
|
||||
import inspect
|
||||
from collections.abc import Coroutine
|
||||
from asyncio import Future
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Generator, Iterable
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Coroutine,
|
||||
Generator,
|
||||
Iterable,
|
||||
Union
|
||||
)
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
|
||||
|
|
@ -34,7 +41,7 @@ def defer_succeed(result) -> Deferred:
|
|||
"""Same as twisted.internet.defer.succeed but delay calling callback until
|
||||
next reactor loop
|
||||
|
||||
It delays by 100ms so reactor has a chance to go trough readers and writers
|
||||
It delays by 100ms so reactor has a chance to go through readers and writers
|
||||
before attending pending delayed calls, so do not set delay to zero.
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
|
|||
return defer.fail(result)
|
||||
else:
|
||||
return defer.succeed(result)
|
||||
|
||||
|
||||
def deferred_to_future(d: Deferred) -> Future:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return an :class:`asyncio.Future` object that wraps *d*.
|
||||
|
||||
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
|
||||
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
|
||||
callables defined as coroutines <coroutine-support>`, you can only await on
|
||||
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
|
||||
allows you to wait on them::
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
additional_response = await deferred_to_future(d)
|
||||
"""
|
||||
return d.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return *d* as an object that can be awaited from a :ref:`Scrapy callable
|
||||
defined as a coroutine <coroutine-support>`.
|
||||
|
||||
What you can await in Scrapy callables defined as coroutines depends on the
|
||||
value of :setting:`TWISTED_REACTOR`:
|
||||
|
||||
- When not using the asyncio reactor, you can only await on
|
||||
:class:`~twisted.internet.defer.Deferred` objects.
|
||||
|
||||
- When :ref:`using the asyncio reactor <install-asyncio>`, you can only
|
||||
await on :class:`asyncio.Future` objects.
|
||||
|
||||
If you want to write code that uses ``Deferred`` objects but works with any
|
||||
reactor, use this function on all ``Deferred`` objects::
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
extra_response = await maybe_deferred_to_future(d)
|
||||
"""
|
||||
if not is_asyncio_reactor_installed():
|
||||
return d
|
||||
else:
|
||||
return deferred_to_future(d)
|
||||
|
|
|
|||
|
|
@ -124,8 +124,9 @@ def _get_handler(settings):
|
|||
""" Return a log handler object according to settings """
|
||||
filename = settings.get('LOG_FILE')
|
||||
if filename:
|
||||
mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
|
||||
encoding = settings.get('LOG_ENCODING')
|
||||
handler = logging.FileHandler(filename, encoding=encoding)
|
||||
handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
|
||||
elif settings.getbool('LOG_ENABLED'):
|
||||
handler = logging.StreamHandler()
|
||||
else:
|
||||
|
|
@ -142,7 +143,7 @@ def _get_handler(settings):
|
|||
return handler
|
||||
|
||||
|
||||
def log_scrapy_info(settings):
|
||||
def log_scrapy_info(settings: Settings) -> None:
|
||||
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
|
||||
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
|
||||
versions = [
|
||||
|
|
@ -151,6 +152,9 @@ def log_scrapy_info(settings):
|
|||
if name != "Scrapy"
|
||||
]
|
||||
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
|
||||
|
||||
|
||||
def log_reactor_info() -> None:
|
||||
from twisted.internet import reactor
|
||||
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
|
||||
from twisted.internet import asyncioreactor
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue