RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into mypy-imports

This commit is contained in:
Andrey Rakhmatullin 2023-02-11 22:01:38 +04:00
parent b1f33a68ac b9f52feaa7
commit 824641e349
288 changed files with 3337 additions and 2537 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.7.1
current_version = 2.8.0
commit = True
tag = True
tag_name = {new_version}

15
.github/workflows/checks.yml vendored
View File

@ -8,12 +8,6 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.11"
env:
TOXENV: security
- python-version: "3.11"
env:
TOXENV: flake8
- python-version: "3.11"
env:
TOXENV: pylint
@ -26,9 +20,6 @@ jobs:
- python-version: "3.11"
env:
TOXENV: twinecheck
- python-version: "3.11"
env:
TOXENV: black
steps:
- uses: actions/checkout@v3
@ -43,3 +34,9 @@ jobs:
run: |
pip install -U tox
tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: pre-commit/action@v3.0.0

40
.github/workflows/publish.yml vendored
View File

@ -1,31 +1,21 @@
name: Publish
on: [push]
on:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
jobs:
publish:
runs-on: ubuntu-latest
if: startsWith(github.event.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Check Tag
id: check-release-tag
run: |
if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
echo ::set-output name=release_tag::true
fi
- name: Publish to PyPI
if: steps.check-release-tag.outputs.release_tag == 'true'
run: |
pip install --upgrade build twine
python -m build
export TWINE_USERNAME=__token__
export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
twine upload dist/*
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.11
- run: |
pip install --upgrade build twine
python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@v1.6.4
with:
password: ${{ secrets.PYPI_TOKEN }}

2
.isort.cfg Normal file
View File

@ -0,0 +1,2 @@
[settings]
profile = black

19
.pre-commit-config.yaml
View File

@ -5,15 +5,20 @@ repos:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
rev: 5.0.4 # 6.0.0 drops Python 3.7 support
hooks:
- id: flake8
- repo: https://github.com/PyCQA/pylint
rev: v2.15.6
hooks:
- id: pylint
args: [conftest.py, docs, extras, scrapy, setup.py, tests]
- repo: https://github.com/psf/black.git
rev: 22.12.0
rev: 23.1.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.11.5 # 5.12 drops Python 3.7 support
hooks:
- id: isort
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.13.0
hooks:
- id: blacken-docs
additional_dependencies:
- black==23.1.0

1
conftest.py
View File

@ -4,7 +4,6 @@ import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
from tests.keys import generate_keys

3
docs/_ext/scrapydocs.py
View File

@ -1,7 +1,8 @@
from operator import itemgetter
from docutils.parsers.rst.roles import set_classes
from docutils import nodes
from docutils.parsers.rst import Directive
from docutils.parsers.rst.roles import set_classes
from sphinx.util.nodes import make_refnode

20
docs/contributing.rst
View File

@ -49,7 +49,7 @@ guidelines when you're going to report a new bug.
(use "scrapy" tag).
* check the `open issues`_ to see if the issue has already been reported. If it
has, don't dismiss the report, but check the ticket history and comments. If
has, don't dismiss the report, but check the ticket history and comments. If
you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
@ -169,7 +169,7 @@ Coding style
Please follow these coding conventions when writing code for inclusion in
Scrapy:
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
@ -179,29 +179,31 @@ Scrapy:
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
.. _scrapy-pre-commit:
Pre-commit
==========
We use `pre-commit`_ to automatically address simple code issues before every
We use `pre-commit`_ to automatically address simple code issues before every
commit.
.. _pre-commit: https://pre-commit.com/
Before you start writing a patch:
After your create a local clone of your fork of the Scrapy repository:
#. `Install pre-commit <https://pre-commit.com/#installation>`_.
#. On the root of your local clone of the Scrapy repository, run the following
#. On the root of your local clone of the Scrapy repository, run the following
command:
.. code-block:: bash
pre-commit install
Now pre-commit will check your changes every time you create a Git commit. Upon
finding issues, pre-commit aborts your commit, and either fixes those issues
automatically, or only reports them to you. If it fixes those issues
automatically, creating your commit again should succeed. Otherwise, you may
Now pre-commit will check your changes every time you create a Git commit. Upon
finding issues, pre-commit aborts your commit, and either fixes those issues
automatically, or only reports them to you. If it fixes those issues
automatically, creating your commit again should succeed. Otherwise, you may
need to address the corresponding issues manually first.
.. _documentation-policies:

45
docs/faq.rst
View File

@ -35,8 +35,9 @@ for parsing HTML responses in Scrapy callbacks.
You just have to feed the response's body into a ``BeautifulSoup`` object
and extract whatever data you need from it.
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser::
Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:
.. code-block:: python
from bs4 import BeautifulSoup
import scrapy
@ -45,17 +46,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
class ExampleSpider(scrapy.Spider):
name = "example"
allowed_domains = ["example.com"]
start_urls = (
'http://www.example.com/',
)
start_urls = ("http://www.example.com/",)
def parse(self, response):
# use lxml to get decent HTML parsing speed
soup = BeautifulSoup(response.text, 'lxml')
yield {
"url": response.url,
"title": soup.h1.string
}
soup = BeautifulSoup(response.text, "lxml")
yield {"url": response.url, "title": soup.h1.string}
.. note::
@ -109,11 +105,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient
in most cases.
If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
setting the following settings:
.. code-block:: python
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
While pending requests are below the configured values of
:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
@ -159,11 +157,13 @@ See also other suggestions at `StackOverflow`_.
.. note:: Remember to disable
:class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
your custom implementation::
your custom implementation:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
'myproject.middlewares.CustomOffsiteMiddleware': 500,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
"myproject.middlewares.CustomOffsiteMiddleware": 500,
}
.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
@ -235,11 +235,12 @@ What does the response status code 999 means?
999 is a custom response status code used by Yahoo sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
higher) in your spider::
higher) in your spider:
.. code-block:: python
class MySpider(CrawlSpider):
name = 'myspider'
name = "myspider"
download_delay = 2
@ -351,19 +352,21 @@ How to split an item into multiple items in an item pipeline?
input item. :ref:`Create a spider middleware <custom-spider-middleware>`
instead, and use its
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method for this purpose. For example::
method for this purpose. For example:
.. code-block:: python
from copy import deepcopy
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
for _ in range(adapter["multiply_by"]):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?

14
docs/intro/overview.rst
View File

@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
example of a Scrapy Spider using the simplest way to run a spider.
Here's the code for a spider that scrapes famous quotes from website
https://quotes.toscrape.com, following the pagination::
https://quotes.toscrape.com, following the pagination:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/tag/humor/',
"https://quotes.toscrape.com/tag/humor/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'author': quote.xpath('span/small/text()').get(),
'text': quote.css('span.text::text').get(),
"author": quote.xpath("span/small/text()").get(),
"text": quote.css("span.text::text").get(),
}
next_page = response.css('li.next a::attr("href")').get()

265
docs/intro/tutorial.rst
View File

@ -83,7 +83,9 @@ optionally how to follow links in the pages, and how to parse the downloaded
page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:
.. code-block:: python
from pathlib import Path
@ -95,17 +97,17 @@ This is the code for our first Spider. Save it in a file named
def start_requests(self):
urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
self.log(f"Saved file {filename}")
As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
@ -177,7 +179,9 @@ that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
for your spider.
.. code-block:: python
from pathlib import Path
@ -187,13 +191,13 @@ for your spider::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
filename = f"quotes-{page}.html"
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
@ -245,8 +249,10 @@ object:
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
>>> response.css('title')
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
.. code-block:: pycon
>>> response.css("title")
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@ -256,42 +262,54 @@ data.
To extract the text from the title above, you can do:
>>> response.css('title::text').getall()
['Quotes to Scrape']
.. code-block:: pycon
>>> response.css("title::text").getall()
['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
``<title>`` element. If we don't specify ``::text``, we'd get the full title
element, including its tags:
>>> response.css('title').getall()
['<title>Quotes to Scrape</title>']
.. code-block:: pycon
>>> response.css("title").getall()
['<title>Quotes to Scrape</title>']
The other thing is that the result of calling ``.getall()`` is a list: it is
possible that a selector returns more than one result, so we extract them all.
When you know you just want the first result, as in this case, you can do:
>>> response.css('title::text').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text").get()
'Quotes to Scrape'
As an alternative, you could've written:
>>> response.css('title::text')[0].get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.css("title::text")[0].get()
'Quotes to Scrape'
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
raise an :exc:`IndexError` exception if there are no results::
raise an :exc:`IndexError` exception if there are no results:
>>> response.css('noelement')[0].get()
.. code-block:: pycon
>>> response.css("noelement")[0].get()
Traceback (most recent call last):
...
IndexError: list index out of range
You might want to use ``.get()`` directly on the
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
if there are no results::
if there are no results:
>>> response.css("noelement").get()
.. code-block:: pycon
>>> response.css("noelement").get()
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
@ -302,12 +320,14 @@ Besides the :meth:`~scrapy.selector.SelectorList.getall` and
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']
.. code-block:: pycon
>>> response.css("title::text").re(r"Quotes.*")
['Quotes to Scrape']
>>> response.css("title::text").re(r"Q\w+")
['Quotes']
>>> response.css("title::text").re(r"(\w+) to (\w+)")
['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
@ -325,10 +345,12 @@ XPath: a brief intro
Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath('//title/text()').get()
'Quotes to Scrape'
.. code-block:: pycon
>>> response.xpath("//title")
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
>>> response.xpath("//title/text()").get()
'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@ -385,33 +407,41 @@ we want::
We get a list of selectors for the quote HTML elements with:
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
.. code-block:: pycon
>>> response.css("div.quote")
[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
variable, so that we can run our CSS selectors directly on a particular quote:
>>> quote = response.css("div.quote")[0]
.. code-block:: pycon
>>> quote = response.css("div.quote")[0]
Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created:
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
.. code-block:: pycon
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
'Albert Einstein'
Given that the tags are a list of strings, we can use the ``.getall()`` method
to get all of them:
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. code-block:: pycon
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
.. invisible-code-block: python
@ -420,14 +450,17 @@ to get all of them:
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary:
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
.. code-block:: pycon
>>> for quote in response.css("div.quote"):
... text = quote.css("span.text::text").get()
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
...
{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
...
Extracting data in our spider
-----------------------------
@ -438,7 +471,9 @@ extraction logic above into our spider.
A Scrapy spider typically generates many dictionaries containing the data
extracted from the page. To do that, we use the ``yield`` Python keyword
in the callback, as you can see below::
in the callback, as you can see below:
.. code-block:: python
import scrapy
@ -446,16 +481,16 @@ in the callback, as you can see below::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
'https://quotes.toscrape.com/page/2/',
"https://quotes.toscrape.com/page/1/",
"https://quotes.toscrape.com/page/2/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
If you run this spider, it will output the extracted data with the log::
@ -533,17 +568,23 @@ This gets the anchor element, but we want the attribute ``href``. For that,
Scrapy supports a CSS extension that lets you select the attribute contents,
like this:
>>> response.css('li.next a::attr(href)').get()
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a::attr(href)").get()
'/page/2/'
There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more):
>>> response.css('li.next a').attrib['href']
'/page/2/'
.. code-block:: pycon
>>> response.css("li.next a").attrib["href"]
'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
page, extracting data from it:
.. code-block:: python
import scrapy
@ -551,18 +592,18 @@ page, extracting data from it::
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@ -594,7 +635,9 @@ A shortcut for creating Requests
--------------------------------
As a shortcut for creating Request objects you can use
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
:meth:`response.follow <scrapy.http.TextResponse.follow>`:
.. code-block:: python
import scrapy
@ -602,18 +645,18 @@ As a shortcut for creating Request objects you can use
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = [
'https://quotes.toscrape.com/page/1/',
"https://quotes.toscrape.com/page/1/",
]
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('span small::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("span small::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
@ -622,57 +665,67 @@ need to call urljoin. Note that ``response.follow`` just returns a Request
instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
this selector should extract necessary attributes:
for href in response.css('ul.pager a::attr(href)'):
.. code-block:: python
for href in response.css("ul.pager a::attr(href)"):
yield response.follow(href, callback=self.parse)
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
attribute automatically. So the code can be shortened further:
for a in response.css('ul.pager a'):
.. code-block:: python
for a in response.css("ul.pager a"):
yield response.follow(a, callback=self.parse)
To create multiple requests from an iterable, you can use
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead:
anchors = response.css('ul.pager a')
.. code-block:: python
anchors = response.css("ul.pager a")
yield from response.follow_all(anchors, callback=self.parse)
or, shortening it further::
or, shortening it further:
yield from response.follow_all(css='ul.pager a', callback=self.parse)
.. code-block:: python
yield from response.follow_all(css="ul.pager a", callback=self.parse)
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
this time for scraping author information:
.. code-block:: python
import scrapy
class AuthorSpider(scrapy.Spider):
name = 'author'
name = "author"
start_urls = ['https://quotes.toscrape.com/']
start_urls = ["https://quotes.toscrape.com/"]
def parse(self, response):
author_page_links = response.css('.author + a')
author_page_links = response.css(".author + a")
yield from response.follow_all(author_page_links, self.parse_author)
pagination_links = response.css('li.next a')
pagination_links = response.css("li.next a")
yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
return response.css(query).get(default='').strip()
return response.css(query).get(default="").strip()
yield {
'name': extract_with_css('h3.author-title::text'),
'birthdate': extract_with_css('.author-born-date::text'),
'bio': extract_with_css('.author-description::text'),
"name": extract_with_css("h3.author-title::text"),
"birthdate": extract_with_css(".author-born-date::text"),
"bio": extract_with_css(".author-description::text"),
}
This spider will start from the main page, it will follow all the links to the
@ -720,7 +773,9 @@ spider attributes by default.
In this example, the value provided for the ``tag`` argument will be available
via ``self.tag``. You can use this to make your spider fetch only quotes
with a specific tag, building the URL based on the argument::
with a specific tag, building the URL based on the argument:
.. code-block:: python
import scrapy
@ -729,20 +784,20 @@ with a specific tag, building the URL based on the argument::
name = "quotes"
def start_requests(self):
url = 'https://quotes.toscrape.com/'
tag = getattr(self, 'tag', None)
url = "https://quotes.toscrape.com/"
tag = getattr(self, "tag", None)
if tag is not None:
url = url + 'tag/' + tag
url = url + "tag/" + tag
yield scrapy.Request(url, self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
}
next_page = response.css('li.next a::attr(href)').get()
next_page = response.css("li.next a::attr(href)").get()
if next_page is not None:
yield response.follow(next_page, self.parse)

218
docs/news.rst
View File

@ -3,6 +3,212 @@
Release notes
=============
.. _release-2.8.0:
Scrapy 2.8.0 (2023-02-02)
-------------------------
This is a maintenance release, with minor features, bug fixes, and cleanups.
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now
been removed. Use the :meth:`~io.BufferedIOBase.read1` method of
:class:`~gzip.GzipFile` instead.
(:issue:`5719`)
- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy
2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode`
instead.
(:issue:`5719`)
- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy
2.0, has now been removed. Use
:meth:`~scrapy.utils.python.MutableChain.__next__` instead.
(:issue:`5719`)
- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated
in Scrapy 2.0, has now been removed. Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead.
(:issue:`5720`)
- Support for using environment variables prefixed with ``SCRAPY_`` to
override settings, deprecated in Scrapy 2.0, has now been removed.
(:issue:`5724`)
- Support for the ``noconnect`` query string argument in proxy URLs,
deprecated in Scrapy 2.0, has now been removed. We expect proxies that used
to need it to work fine without it.
(:issue:`5731`)
- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy
2.3, has now been removed.
(:issue:`5719`)
- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4,
has now been removed.
(:issue:`5719`)
Deprecations
~~~~~~~~~~~~
- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated.
(:issue:`5368`, :issue:`5489`)
- :meth:`ImagesPipeline.convert_image
<scrapy.pipelines.images.ImagesPipeline.convert_image>` must now accept a
``response_body`` parameter.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
New features
~~~~~~~~~~~~
- Applied black_ coding style to files generated with the
:command:`genspider` and :command:`startproject` commands.
(:issue:`5809`, :issue:`5814`)
.. _black: https://black.readthedocs.io/en/stable/
- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the
``settings.py`` file that the :command:`startproject` command generates.
With this value, JSON exports won’t force the use of escape sequences for
non-ASCII characters.
(:issue:`5797`, :issue:`5800`)
- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the
peak memory usage during checks, and the binary unit MiB is now used to
avoid confusion.
(:issue:`5717`, :issue:`5722`, :issue:`5727`)
- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set
to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from
``None``, as the latter indicates that the default spider callback
(:meth:`~scrapy.Spider.parse`) is to be used.
(:issue:`5798`)
Bug fixes
~~~~~~~~~
- Enabled unsafe legacy SSL renegotiation to fix access to some outdated
websites.
(:issue:`5491`, :issue:`5790`)
- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and
better.
(:issue:`5386`, :issue:`5406`)
- Fixed the :meth:`finish_exporting` method of :ref:`item exporters
<topics-exporters>` not being called for empty files.
(:issue:`5537`, :issue:`5758`)
- Fixed HTTP/2 responses getting only the last value for a header when
multiple headers with the same name are received.
(:issue:`5777`)
- Fixed an exception raised by the :command:`shell` command on some cases
when :ref:`using asyncio <using-asyncio>`.
(:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`,
:issue:`5771`)
- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments
(``cb_kwargs``) added to a request in the ``process_request`` callback of a
:class:`~scrapy.spiders.Rule` will no longer be ignored.
(:issue:`5699`)
- The :ref:`images pipeline <images-pipeline>` no longer re-encodes JPEG
files.
(:issue:`3055`, :issue:`3689`, :issue:`4753`)
- Fixed the handling of transparent WebP images by the :ref:`images pipeline
<images-pipeline>`.
(:issue:`3072`, :issue:`5766`, :issue:`5767`)
- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT``
(Ctrl+C).
(:issue:`2918`)
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
with ``unique=False`` no longer filters out links that have identical URL
*and* text.
(:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`)
- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now
ignores URL protocols that do not support ``robots.txt`` (``data://``,
``file://``).
(:issue:`5807`)
- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6.
(:issue:`5753`, :issue:`5754`)
- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**``
line.
(:issue:`5709`, :issue:`5711`, :issue:`5712`)
- Made the active project indication in the output of :ref:`commands
<topics-commands>` more clear.
(:issue:`5715`)
Documentation
~~~~~~~~~~~~~
- Documented how to :ref:`debug spiders from Visual Studio Code
<debug-vscode>`.
(:issue:`5721`)
- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency.
(:issue:`5083`, :issue:`5540`)
- Improved consistency.
(:issue:`5761`)
- Fixed typos.
(:issue:`5714`, :issue:`5744`, :issue:`5764`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Applied :ref:`black coding style <coding-style>`, sorted import statements,
and introduced :ref:`pre-commit <scrapy-pre-commit>`.
(:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`,
:issue:`5810`)
- Switched from :mod:`os.path` to :mod:`pathlib`.
(:issue:`4916`, :issue:`4497`, :issue:`5682`)
- Addressed many issues reported by Pylint.
(:issue:`5677`)
- Improved code readability.
(:issue:`5736`)
- Improved package metadata.
(:issue:`5768`)
- Removed direct invocations of ``setup.py``.
(:issue:`5774`, :issue:`5776`)
- Removed unnecessary :class:`~collections.OrderedDict` usages.
(:issue:`5795`)
- Removed unnecessary ``__str__`` definitions.
(:issue:`5150`)
- Removed obsolete code and comments.
(:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`)
- Fixed test and CI issues.
(:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`,
:issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`,
:issue:`5786`)
.. _release-2.7.1:
Scrapy 2.7.1 (2022-11-02)
@ -2417,11 +2623,13 @@ Backward-incompatible changes
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
into lists:
>>> item = MyItem()
>>> item['field'] = 'value1'
>>> loader = ItemLoader(item=item)
>>> item['field']
['value1']
.. code-block:: pycon
>>> item = MyItem()
>>> item["field"] = "value1"
>>> loader = ItemLoader(item=item)
>>> item["field"]
['value1']
This is needed to allow adding values to existing fields
(``loader.add_value('field', 'value2')``).

14
docs/topics/api.rst
View File

@ -132,16 +132,14 @@ Settings API
precedence over lesser ones when setting and retrieving values in the
:class:`~scrapy.settings.Settings` class.
.. highlight:: python
::
.. code-block:: python
SETTINGS_PRIORITIES = {
'default': 0,
'command': 10,
'project': 20,
'spider': 30,
'cmdline': 40,
"default": 0,
"command": 10,
"project": 20,
"spider": 30,
"cmdline": 40,
}
For a detailed explanation on each settings sources, see:

6
docs/topics/asyncio.rst
View File

@ -106,12 +106,14 @@ Enforcing asyncio as a requirement
If you are writing a :ref:`component <topics-components>` that requires asyncio
to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
example::
example:
.. code-block:: python
from scrapy.utils.reactor import is_asyncio_reactor_installed
class MyComponent:
class MyComponent:
def __init__(self):
if not is_asyncio_reactor_installed():
raise ValueError(

40
docs/topics/broad-crawls.rst
View File

@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ
It works best during single-domain crawl. It does not work well with crawling
many different domains in parallel
To apply the recommended priority queue use::
To apply the recommended priority queue use:
SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
.. code-block:: python
SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
.. _broad-crawls-concurrency:
@ -71,7 +73,9 @@ many different domains in parallel, so you will want to increase it. How much
to increase it will depend on how much CPU and memory your crawler will have
available.
A good starting point is ``100``::
A good starting point is ``100``:
.. code-block:: python
CONCURRENT_REQUESTS = 100
@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of
threads handling DNS queries. The DNS queue will be processed faster speeding
up establishing of connection and crawling overall.
To increase maximum thread pool size use::
To increase maximum thread pool size use:
.. code-block:: python
REACTOR_THREADPOOL_MAXSIZE = 20
@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
To set the log level use:
LOG_LEVEL = 'INFO'
.. code-block:: python
LOG_LEVEL = "INFO"
Disable cookies
===============
@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve
performance by saving some CPU cycles and reducing the memory footprint of your
Scrapy crawler.
To disable cookies use::
To disable cookies use:
.. code-block:: python
COOKIES_ENABLED = False
@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout
error which gets retried many times, unnecessarily, preventing crawler capacity
to be reused for other domains.
To disable retries use::
To disable retries use:
.. code-block:: python
RETRY_ENABLED = False
@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the
case for broad crawls) reduce the download timeout so that stuck requests are
discarded quickly and free up capacity to process the next ones.
To reduce the download timeout use::
To reduce the download timeout use:
.. code-block:: python
DOWNLOAD_TIMEOUT = 15
@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of
request constant per crawl batch, otherwise redirect loops may cause the
crawler to dedicate too many resources on any specific domain.
To disable redirects use::
To disable redirects use:
.. code-block:: python
REDIRECT_ENABLED = False
@ -179,7 +195,9 @@ Pages can indicate it in two ways:
"main", "index" website pages.
Scrapy handles (1) automatically; to handle (2) enable
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`::
:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`:
.. code-block:: python
AJAXCRAWL_ENABLED = True

17
docs/topics/commands.rst
View File

@ -617,7 +617,7 @@ Example:
.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
COMMANDS_MODULE = "mybot.commands"
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
@ -636,10 +636,11 @@ The following example adds ``my_command`` command:
from setuptools import setup, find_packages
setup(name='scrapy-mymodule',
entry_points={
'scrapy.commands': [
'my_command=my_scrapy_module.commands:MyCommand',
],
},
)
setup(
name="scrapy-mymodule",
entry_points={
"scrapy.commands": [
"my_command=my_scrapy_module.commands:MyCommand",
],
},
)

8
docs/topics/components.rst
View File

@ -66,16 +66,18 @@ version mismatch, while :exc:`ValueError` may be better if the issue is the
value of a setting.
If your requirement is a minimum Scrapy version, you may use
:attr:`scrapy.__version__` to enforce your requirement. For example::
:attr:`scrapy.__version__` to enforce your requirement. For example:
.. code-block:: python
from pkg_resources import parse_version
import scrapy
class MyComponent:
class MyComponent:
def __init__(self):
if parse_version(scrapy.__version__) < parse_version('2.7'):
if parse_version(scrapy.__version__) < parse_version("2.7"):
raise RuntimeError(
f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
f"later, which allow defining the process_spider_output "

38
docs/topics/contracts.rst
View File

@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts.
This allows you to test each callback of your spider by hardcoding a sample url
and check various constraints for how the callback processes the response. Each
contract is prefixed with an ``@`` and included in the docstring. See the
following example::
following example:
.. code-block:: python
def parse(self, response):
""" This function parses a sample response. Some contracts are mingled
"""
This function parses a sample response. Some contracts are mingled
with this docstring.
@url http://www.amazon.com/s?field-keywords=selfish+gene
@ -64,11 +67,13 @@ Custom Contracts
If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::
:setting:`SPIDER_CONTRACTS` setting:
.. code-block:: python
SPIDER_CONTRACTS = {
'myproject.contracts.ResponseCheck': 10,
'myproject.contracts.ItemValidate': 10,
"myproject.contracts.ResponseCheck": 10,
"myproject.contracts.ItemValidate": 10,
}
Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
@ -111,22 +116,26 @@ Raise :class:`~scrapy.exceptions.ContractFail` from
.. autoclass:: scrapy.exceptions.ContractFail
Here is a demo contract which checks the presence of a custom header in the
response received::
response received:
.. code-block:: python
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
class HasHeaderContract(Contract):
""" Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
Demo contract which checks the presence of a custom header
@has_header X-CustomHeader
"""
name = 'has_header'
name = "has_header"
def pre_process(self, response):
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
raise ContractFail("X-CustomHeader not present")
.. _detecting-contract-check-runs:
@ -135,14 +144,17 @@ Detecting check runs
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
your spiders or your settings when ``scrapy check`` is used:
.. code-block:: python
import os
import scrapy
class ExampleSpider(scrapy.Spider):
name = 'example'
name = "example"
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
if os.environ.get("SCRAPY_CHECK"):
pass # Do some scraper adjustments when a check is running

29
docs/topics/coroutines.rst
View File

@ -58,49 +58,58 @@ There are several use cases for coroutines in Scrapy.
Code that would return Deferreds when written for previous Scrapy versions,
such as downloader middlewares and signal handlers, can be rewritten to be
shorter and cleaner::
shorter and cleaner:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
adapter = ItemAdapter(item)
adapter['field'] = data
adapter["field"] = data
return item
def process_item(self, item, spider):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
becomes:
.. code-block:: python
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
adapter["field"] = await db.get_some_data(adapter["id"])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
This means you can use many useful Python libraries providing such code:
.. code-block:: python
class MySpiderDeferred(Spider):
# ...
async def parse(self, response):
additional_response = await treq.get('https://additional.url')
additional_response = await treq.get("https://additional.url")
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
class MySpiderAsyncio(Spider):
# ...
async def parse(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
async with session.get("https://additional.url") as additional_response:
additional_data = await additional_response.text()
# ... use response and additional_data to yield items and requests
@ -192,7 +201,9 @@ while maintaining support for older Scrapy versions, you may define
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example::
For example:
.. code-block:: python
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):

37
docs/topics/debug.rst
View File

@ -5,21 +5,24 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
Consider the following Scrapy spider below::
Consider the following Scrapy spider below:
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
start_urls = (
'http://example.com/page1',
'http://example.com/page2',
)
"http://example.com/page1",
"http://example.com/page2",
)
def parse(self, response):
# <processing code not shown>
# collect `item_urls`
# collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
@ -28,7 +31,9 @@ Consider the following Scrapy spider below::
item = MyItem()
# populate `item` fields
# and extract item_details_url
yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
yield scrapy.Request(
item_details_url, self.parse_details, cb_kwargs={"item": item}
)
def parse_details(self, response, item):
# populate more `item` fields
@ -103,10 +108,13 @@ showing the response received and the output. How to debug the situation when
.. highlight:: python
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
:ref:`topics-shell-inspect-response`):
.. code-block:: python
from scrapy.shell import inspect_response
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
@ -121,10 +129,13 @@ Open in browser
Sometimes you just want to see how a certain response looks in a browser, you
can use the ``open_in_browser`` function for that. Here is an example of how
you would use it::
you would use it:
.. code-block:: python
from scrapy.utils.response import open_in_browser
def parse_details(self, response):
if "item name" not in response.body:
open_in_browser(response)
@ -138,19 +149,23 @@ Logging
Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
available in all future runs should they be necessary again:
.. code-block:: python
def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
else:
self.logger.warning('No item received for %s', response.url)
self.logger.warning("No item received for %s", response.url)
For more information, check the :ref:`topics-logging` section.
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
.. _debug-vscode:
Visual Studio Code
==================

35
docs/topics/developer-tools.rst
View File

@ -94,8 +94,10 @@ Then, back to your web browser, right-click on the ``span`` tag, select
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
.. code-block:: pycon
>>> response.xpath("/html/body/div/div[2]/div[1]/div[1]/span[1]/text()").getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@ -124,11 +126,13 @@ With this knowledge we can refine our XPath: Instead of a path to follow,
we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
.. code-block:: pycon
>>> response.xpath('//span[has-class("text")]/text()').getall()
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@ -237,17 +241,19 @@ on the request and open ``Open in new tab`` to get a better overview.
:alt: JSON-object returned from the quotes.toscrape API
With this response we can now easily parse the JSON-object and
also request each page to get every quote on the site::
also request each page to get every quote on the site:
.. code-block:: python
import scrapy
import json
class QuoteSpider(scrapy.Spider):
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
name = "quote"
allowed_domains = ["quotes.toscrape.com"]
page = 1
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
def parse(self, response):
data = json.loads(response.text)
@ -275,7 +281,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
format, by right-clicking on each of them in the network tool and using the
:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
request::
request:
.. code-block:: python
from scrapy import Request
@ -286,7 +294,8 @@ request::
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
"zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
"I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'"
)
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`

51
docs/topics/downloader-middleware.rst
View File

@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the
:setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class paths and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
}
The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the
@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a built-in middleware (the ones defined in
:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
as its value. For example, if you want to disable the user-agent middleware::
as its value. For example, if you want to disable the user-agent middleware:
.. code-block:: python
DOWNLOADER_MIDDLEWARES = {
'myproject.middlewares.CustomDownloaderMiddleware': 543,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
"myproject.middlewares.CustomDownloaderMiddleware": 543,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -226,20 +230,25 @@ There is support for keeping multiple cookie sessions per spider by using the
:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
(session), but you can pass an identifier to use different ones.
For example::
For example:
.. code-block:: python
for i, url in enumerate(urls):
yield scrapy.Request(url, meta={'cookiejar': i},
callback=self.parse_page)
yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page)
Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep
passing it along on subsequent requests. For example::
passing it along on subsequent requests. For example:
.. code-block:: python
def parse_page(self, response):
# do some processing
return scrapy.Request("http://www.example.com/otherpage",
meta={'cookiejar': response.meta['cookiejar']},
callback=self.parse_other_page)
return scrapy.Request(
"http://www.example.com/otherpage",
meta={"cookiejar": response.meta["cookiejar"]},
callback=self.parse_other_page,
)
.. setting:: COOKIES_ENABLED
@ -339,16 +348,18 @@ HttpAuthMiddleware
domain of the first request, which will work for some spiders but not
for others. In the future the middleware will produce an error instead.
Example::
Example:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class SomeIntranetSiteSpider(CrawlSpider):
http_user = 'someuser'
http_pass = 'somepass'
http_auth_domain = 'intranet.example.com'
name = 'intranet.example.com'
class SomeIntranetSiteSpider(CrawlSpider):
http_user = "someuser"
http_pass = "somepass"
http_auth_domain = "intranet.example.com"
name = "intranet.example.com"
# .. rest of the spider code omitted ...
@ -792,7 +803,9 @@ If you want to handle some redirect status codes in your spider, you can
specify these in the ``handle_httpstatus_list`` spider attribute.
For example, if you want the redirect middleware to ignore 301 and 302
responses (and pass them through to your spider) you can do this::
responses (and pass them through to your spider) you can do this:
.. code-block:: python
class MySpider(CrawlSpider):
handle_httpstatus_list = [301, 302]

55
docs/topics/dynamic-content.rst
View File

@ -119,16 +119,20 @@ data from it depends on the type of response:
<topics-selectors>` as usual.
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
:attr:`response.text <scrapy.http.TextResponse.text>`:
.. code-block:: python
data = json.loads(response.text)
If the desired data is inside HTML or XML code embedded within JSON data,
you can load that HTML or XML code into a
:class:`~scrapy.Selector` and then
:ref:`use it <topics-selectors>` as usual::
:ref:`use it <topics-selectors>` as usual:
selector = Selector(data['html'])
.. code-block:: python
selector = Selector(data["html"])
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
@ -179,10 +183,12 @@ data from it:
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
>>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
>>> json_data = response.css('script::text').re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
.. code-block:: pycon
>>> pattern = r"\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n"
>>> json_data = response.css("script::text").re_first(pattern)
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
@ -190,11 +196,13 @@ data from it:
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
.. code-block:: pycon
>>> import chompjs
>>> javascript = response.css("script::text").get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -202,14 +210,16 @@ data from it:
For example, if the JavaScript code contains
``var data = {field: "value"};`` you can extract that data as follows:
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css('script::text').get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. code-block:: pycon
>>> import js2xml
>>> import lxml.etree
>>> from parsel import Selector
>>> javascript = response.css("script::text").get()
>>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding="unicode")
>>> selector = Selector(text=xml)
>>> selector.css('var[name="data"]').get()
'<var name="data"><object><property name="field"><string>value</string></property></object></var>'
.. _topics-javascript-rendering:
@ -250,11 +260,14 @@ automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider::
The following is a simple snippet to illustrate its usage within a Scrapy spider:
.. code-block:: python
import scrapy
from playwright.async_api import async_playwright
class PlaywrightSpider(scrapy.Spider):
name = "playwright"
start_urls = ["data:,"] # avoid using the default Scrapy downloader

20
docs/topics/email.rst
View File

@ -19,19 +19,31 @@ Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
the standard ``__init__`` method::
the standard ``__init__`` method:
.. code-block:: python
from scrapy.mail import MailSender
mailer = MailSender()
Or you can instantiate it passing a Scrapy settings object, which will respect
the :ref:`settings <topics-email-settings>`::
the :ref:`settings <topics-email-settings>`:
.. code-block:: python
mailer = MailSender.from_settings(settings)
And here is how to use it to send an e-mail (without attachments)::
And here is how to use it to send an e-mail (without attachments):
mailer.send(to=["someone@example.com"], subject="Some subject", body="Some body", cc=["another@example.com"])
.. code-block:: python
mailer.send(
to=["someone@example.com"],
subject="Some subject",
body="Some body",
cc=["another@example.com"],
)
MailSender class reference
==========================

8
docs/topics/exceptions.rst
View File

@ -26,11 +26,13 @@ CloseSpider
:param reason: the reason for closing
:type reason: str
For example::
For example:
.. code-block:: python
def parse_page(self, response):
if 'Bandwidth exceeded' in response.body:
raise CloseSpider('bandwidth_exceeded')
if "Bandwidth exceeded" in response.body:
raise CloseSpider("bandwidth_exceeded")
DontCloseSpider
---------------

35
docs/topics/exporters.rst
View File

@ -38,11 +38,14 @@ the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
value of one of their fields:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
@ -56,9 +59,9 @@ value of one of their fields::
def _exporter_for_item(self, item):
adapter = ItemAdapter(item)
year = adapter['year']
year = adapter["year"]
if year not in self.year_to_exporter:
xml_file = open(f'{year}.xml', 'wb')
xml_file = open(f"{year}.xml", "wb")
exporter = XmlItemExporter(xml_file)
exporter.start_exporting()
self.year_to_exporter[year] = (exporter, xml_file)
@ -94,12 +97,16 @@ If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
Example:
.. code-block:: python
import scrapy
def serialize_price(value):
return f'$ {str(value)}'
return f"$ {str(value)}"
class Product(scrapy.Item):
name = scrapy.Field()
@ -115,15 +122,17 @@ customize how your field value will be exported.
Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
after your custom code.
Example::
Example:
.. code-block:: python
from scrapy.exporters import XmlItemExporter
class ProductXmlExporter(XmlItemExporter):
class ProductXmlExporter(XmlItemExporter):
def serialize_field(self, field, name, value):
if name == 'price':
return f'$ {str(value)}'
if name == "price":
return f"$ {str(value)}"
return super().serialize_field(field, name, value)
.. _topics-exporters-reference:
@ -132,10 +141,12 @@ Built-in Item Exporters reference
=================================
Here is a list of the Item Exporters bundled with Scrapy. Some of them contain
output examples, which assume you're exporting these two items::
output examples, which assume you're exporting these two items:
Item(name='Color TV', price='1200')
Item(name='DVD player', price='200')
.. code-block:: python
Item(name="Color TV", price="1200")
Item(name="DVD player", price="200")
BaseItemExporter
----------------

24
docs/topics/extensions.rst
View File

@ -31,11 +31,13 @@ initialization code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
by a string: the full Python path to the extension's class name. For example::
by a string: the full Python path to the extension's class name. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': 500,
'scrapy.extensions.telnet.TelnetConsole': 500,
"scrapy.extensions.corestats.CoreStats": 500,
"scrapy.extensions.telnet.TelnetConsole": 500,
}
@ -64,10 +66,12 @@ Disabling an extension
In order to disable an extension that comes enabled by default (i.e. those
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
``None``. For example::
``None``. For example:
.. code-block:: python
EXTENSIONS = {
'scrapy.extensions.corestats.CoreStats': None,
"scrapy.extensions.corestats.CoreStats": None,
}
Writing your own extension
@ -98,7 +102,9 @@ in the previous section. This extension will log a message every time:
The extension will be enabled through the ``MYEXT_ENABLED`` setting and the
number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting.
Here is the code of such extension::
Here is the code of such extension:
.. code-block:: python
import logging
from scrapy import signals
@ -106,8 +112,8 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging:
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
self.items_scraped = 0
@ -116,11 +122,11 @@ Here is the code of such extension::
def from_crawler(cls, crawler):
# first check if the extension should be enabled and raise
# NotConfigured otherwise
if not crawler.settings.getbool('MYEXT_ENABLED'):
if not crawler.settings.getbool("MYEXT_ENABLED"):
raise NotConfigured
# get the number of items from settings
item_count = crawler.settings.getint('MYEXT_ITEMCOUNT', 1000)
item_count = crawler.settings.getint("MYEXT_ITEMCOUNT", 1000)
# instantiate the extension object
ext = cls(item_count)

69
docs/topics/feed-exports.rst
View File

@ -290,10 +290,11 @@ class, which is the default value of the ``item_filter`` :ref:`feed option <feed
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
method ``accepts`` and taking ``feed_options`` as an argument.
For instance::
For instance:
.. code-block:: python
class MyCustomFilter:
def __init__(self, feed_options):
self.feed_options = feed_options
@ -515,7 +516,7 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
Use ``utf-8`` if you want UTF-8 for JSON too.
.. versionchanged:: VERSION
.. versionchanged:: 2.8
The :command:`startproject` command now sets this setting to
``utf-8`` in the generated ``settings.py`` file.
@ -594,23 +595,27 @@ For a complete list of available values, access the `Canned ACL`_ section on Ama
FEED_STORAGES_BASE
------------------
Default::
Default:
.. code-block:: python
{
'': 'scrapy.extensions.feedexport.FileFeedStorage',
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
"": "scrapy.extensions.feedexport.FileFeedStorage",
"file": "scrapy.extensions.feedexport.FileFeedStorage",
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
}
A dict containing the built-in feed storage backends supported by Scrapy. You
can disable any of these backends by assigning ``None`` to their URI scheme in
:setting:`FEED_STORAGES`. E.g., to disable the built-in FTP storage backend
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_STORAGES = {
'ftp': None,
"ftp": None,
}
.. setting:: FEED_EXPORTERS
@ -628,26 +633,30 @@ serialization formats and the values are paths to :ref:`Item exporter
FEED_EXPORTERS_BASE
-------------------
Default::
Default:
.. code-block:: python
{
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
'jl': 'scrapy.exporters.JsonLinesItemExporter',
'csv': 'scrapy.exporters.CsvItemExporter',
'xml': 'scrapy.exporters.XmlItemExporter',
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
"json": "scrapy.exporters.JsonItemExporter",
"jsonlines": "scrapy.exporters.JsonLinesItemExporter",
"jsonl": "scrapy.exporters.JsonLinesItemExporter",
"jl": "scrapy.exporters.JsonLinesItemExporter",
"csv": "scrapy.exporters.CsvItemExporter",
"xml": "scrapy.exporters.XmlItemExporter",
"marshal": "scrapy.exporters.MarshalItemExporter",
"pickle": "scrapy.exporters.PickleItemExporter",
}
A dict containing the built-in feed exporters supported by Scrapy. You can
disable any of these exporters by assigning ``None`` to their serialization
format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
(without replacement), place this in your ``settings.py``::
(without replacement), place this in your ``settings.py``:
.. code-block:: python
FEED_EXPORTERS = {
'csv': None,
"csv": None,
}
@ -677,7 +686,9 @@ generated:
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
For instance, if your settings include:
.. code-block:: python
FEED_EXPORT_BATCH_ITEM_COUNT = 100
@ -746,16 +757,20 @@ The function signature should be as follows:
For example, to include the :attr:`name <scrapy.Spider.name>` of the
source spider in the feed URI:
#. Define the following function somewhere in your project::
#. Define the following function somewhere in your project:
.. code-block:: python
# myproject/utils.py
def uri_params(params, spider):
return {**params, 'spider_name': spider.name}
return {**params, "spider_name": spider.name}
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:
.. code-block:: python
# myproject/settings.py
FEED_URI_PARAMS = 'myproject.utils.uri_params'
FEED_URI_PARAMS = "myproject.utils.uri_params"
#. Use ``%(spider_name)s`` in your feed URI::

59
docs/topics/item-pipeline.rst
View File

@ -81,19 +81,22 @@ Price validation and dropping items with no prices
Let's take a look at the following hypothetical pipeline that adjusts the
``price`` attribute for those items that do not include VAT
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
contain a price:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline:
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
adapter["price"] = adapter["price"] * self.vat_factor
return item
else:
raise DropItem(f"Missing price in {item}")
@ -104,16 +107,18 @@ Write items to a JSON lines file
The following pipeline stores all scraped items (from all spiders) into a
single ``items.jsonl`` file, containing one item per line serialized in JSON
format::
format:
.. code-block:: python
import json
from itemadapter import ItemAdapter
class JsonWriterPipeline:
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jsonl', 'w')
self.file = open("items.jsonl", "w")
def close_spider(self, spider):
self.file.close()
@ -135,14 +140,16 @@ MongoDB address and database name are specified in Scrapy settings;
MongoDB collection is named after item class.
The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
method and how to clean up the resources properly.
.. code-block:: python
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline:
collection_name = 'scrapy_items'
class MongoPipeline:
collection_name = "scrapy_items"
def __init__(self, mongo_uri, mongo_db):
self.mongo_uri = mongo_uri
@ -151,8 +158,8 @@ method and how to clean up the resources properly.::
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
mongo_uri=crawler.settings.get("MONGO_URI"),
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
def open_spider(self, spider):
@ -183,7 +190,7 @@ render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
.. code-block:: python
import hashlib
from pathlib import Path
@ -191,6 +198,7 @@ item.
import scrapy
from itemadapter import ItemAdapter
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.defer import maybe_deferred_to_future
@ -204,8 +212,10 @@ item.
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await maybe_deferred_to_future(
spider.crawler.engine.download(request, spider)
)
if response.status != 200:
# Error happened, return item.
@ -228,23 +238,24 @@ Duplicates filter
A filter that looks for duplicate items, and drops those items that were
already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
returns multiples items with the same id:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline:
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
if adapter["id"] in self.ids_seen:
raise DropItem(f"Duplicate item found: {item!r}")
else:
self.ids_seen.add(adapter['id'])
self.ids_seen.add(adapter["id"])
return item
@ -252,11 +263,13 @@ Activating an Item Pipeline component
=====================================
To activate an Item Pipeline component you must add its class to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
.. code-block:: python
ITEM_PIPELINES = {
'myproject.pipelines.PricePipeline': 300,
'myproject.pipelines.JsonWriterPipeline': 800,
"myproject.pipelines.PricePipeline": 300,
"myproject.pipelines.JsonWriterPipeline": 800,
}
The integer values you assign to classes in this setting determine the

138
docs/topics/items.rst
View File

@ -76,10 +76,13 @@ make it the most feature-complete item type:
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
Example:
.. code-block:: python
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
@ -102,10 +105,13 @@ Additionally, ``dataclass`` items also allow to:
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
Example::
Example:
.. code-block:: python
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
@ -133,10 +139,13 @@ Additionally, ``attr.s`` items also allow to:
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
Example:
.. code-block:: python
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
@ -152,10 +161,13 @@ Declaring Item subclasses
-------------------------
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
:class:`Field` objects. Here is an example:
.. code-block:: python
import scrapy
class Product(scrapy.Item):
name = scrapy.Field()
price = scrapy.Field()
@ -222,62 +234,68 @@ notice the API is very similar to the :class:`dict` API.
Creating items
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
.. code-block:: pycon
>>> product = Product(name="Desktop PC", price=1000)
>>> print(product)
Product(name='Desktop PC', price=1000)
Getting field values
''''''''''''''''''''
>>> product['name']
Desktop PC
>>> product.get('name')
Desktop PC
.. code-block:: pycon
>>> product['price']
1000
>>> product["name"]
Desktop PC
>>> product.get("name")
Desktop PC
>>> product['last_updated']
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product["price"]
1000
>>> product.get('last_updated', 'not set')
not set
>>> product["last_updated"]
Traceback (most recent call last):
...
KeyError: 'last_updated'
>>> product['lala'] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> product.get("last_updated", "not set")
not set
>>> product.get('lala', 'unknown field')
'unknown field'
>>> product["lala"] # getting unknown field
Traceback (most recent call last):
...
KeyError: 'lala'
>>> 'name' in product # is name field populated?
True
>>> product.get("lala", "unknown field")
'unknown field'
>>> 'last_updated' in product # is last_updated populated?
False
>>> "name" in product # is name field populated?
True
>>> 'last_updated' in product.fields # is last_updated a declared field?
True
>>> "last_updated" in product # is last_updated populated?
False
>>> 'lala' in product.fields # is lala a declared field?
False
>>> "last_updated" in product.fields # is last_updated a declared field?
True
>>> "lala" in product.fields # is lala a declared field?
False
Setting field values
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
today
.. code-block:: pycon
>>> product['lala'] = 'test' # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> product["last_updated"] = "today"
>>> product["last_updated"]
today
>>> product["lala"] = "test" # setting unknown field
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Accessing all populated values
@ -285,11 +303,13 @@ Accessing all populated values
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
.. code-block:: pycon
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
>>> product.keys()
['price', 'name']
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
.. _copying-items:
@ -327,18 +347,20 @@ Other common tasks
Creating dicts from items:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
.. code-block:: pycon
Creating items from dicts:
>>> dict(product) # create a dict from all populated values
{'price': 1000, 'name': 'Desktop PC'}
>>> Product({'name': 'Laptop PC', 'price': 1500})
Product(price=1500, name='Laptop PC')
Creating items from dicts:
>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
>>> Product({"name": "Laptop PC", "price": 1500})
Product(price=1500, name='Laptop PC')
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
Traceback (most recent call last):
...
KeyError: 'Product does not support field: lala'
Extending Item subclasses
@ -347,17 +369,21 @@ Extending Item subclasses
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
For example::
For example:
.. code-block:: python
class DiscountedProduct(Product):
discount_percent = scrapy.Field(serializer=str)
discount_expiration_date = scrapy.Field()
You can also extend field metadata by using the previous field metadata and
appending more values, or changing existing values, like this::
appending more values, or changing existing values, like this:
.. code-block:: python
class SpecificProduct(Product):
name = scrapy.Field(Product.fields['name'], serializer=my_serializer)
name = scrapy.Field(Product.fields["name"], serializer=my_serializer)
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.

6
docs/topics/jobs.rst
View File

@ -51,11 +51,13 @@ loading that attribute from the job directory, when the spider starts and
stops.
Here's an example of a callback that uses the spider state (other spider code
is omitted for brevity)::
is omitted for brevity):
.. code-block:: python
def parse_item(self, response):
# parse item here
self.state['items_count'] = self.state.get('items_count', 0) + 1
self.state["items_count"] = self.state.get("items_count", 0) + 1
Persistence gotchas
===================

96
docs/topics/leaks.rst
View File

@ -70,13 +70,15 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
telnet localhost 6023
>>> prefs()
Live References
.. code-block:: pycon
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
>>> prefs()
Live References
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
FormRequest 878 oldest: 7s ago
As you can see, that report also shows the "age" of the oldest object in each
class. If you're running multiple spiders per process chances are you can
@ -114,7 +116,9 @@ a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
references::
references:
.. code-block:: pycon
>>> prefs()
Live References
@ -134,19 +138,23 @@ generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
Let's check the oldest response:
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest('HtmlResponse')
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
.. code-block:: pycon
>>> from scrapy.utils.trackref import get_oldest
>>> r = get_oldest("HtmlResponse")
>>> r.url
'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
can use the :func:`scrapy.utils.trackref.iter_all` function:
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all('HtmlResponse')]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
.. code-block:: pycon
>>> from scrapy.utils.trackref import iter_all
>>> [r.url for r in iter_all("HtmlResponse")]
['http://www.somenastyspider.com/product.php?pid=123',
'http://www.somenastyspider.com/product.php?pid=584',
...]
Too many spiders?
-----------------
@ -157,8 +165,10 @@ For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclasses). For
example, this won't show any live references to spiders:
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. code-block:: pycon
>>> from scrapy.spiders import Spider
>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@ -216,30 +226,32 @@ If you use ``pip``, you can install muppy with the following command::
Here's an example to view all Python objects available in
the heap using muppy:
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
.. code-block:: pycon
>>> from pympler import muppy
>>> all_objects = muppy.get_objects()
>>> len(all_objects)
28667
>>> from pympler import summary
>>> suml = summary.summarize(all_objects)
>>> summary.print_(suml)
types | # objects | total size
==================================== | =========== | ============
<class 'str | 9822 | 1.10 MB
<class 'dict | 1658 | 856.62 KB
<class 'type | 436 | 443.60 KB
<class 'code | 2974 | 419.56 KB
<class '_io.BufferedWriter | 2 | 256.34 KB
<class 'set | 420 | 159.88 KB
<class '_io.BufferedReader | 1 | 128.17 KB
<class 'wrapper_descriptor | 1130 | 88.28 KB
<class 'tuple | 1304 | 86.57 KB
<class 'weakref | 1013 | 79.14 KB
<class 'builtin_function_or_method | 958 | 67.36 KB
<class 'method_descriptor | 865 | 60.82 KB
<class 'abc.ABCMeta | 62 | 59.96 KB
<class 'list | 446 | 58.52 KB
<class 'int | 1425 | 43.20 KB
For more info about muppy, refer to the `muppy documentation`_.

8
docs/topics/link-extractors.rst
View File

@ -18,7 +18,9 @@ through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
variable in your spider, and use it from your spider callbacks:
.. code-block:: python
def parse(self, response):
for link in self.link_extractor.extract_links(response):
@ -132,7 +134,9 @@ LxmlLinkExtractor
.. highlight:: python
You can use the following function in ``process_value``::
You can use the following function in ``process_value``:
.. code-block:: python
def process_value(value):
m = re.search("javascript:goToPage\('(.*?)'", value)

116
docs/topics/loaders.rst
View File

@ -46,18 +46,21 @@ using a proper processing function.
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
chapter <topics-items>`::
chapter <topics-items>`:
.. code-block:: python
from scrapy.loader import ItemLoader
from myproject.items import Product
def parse(self, response):
l = ItemLoader(item=Product(), response=response)
l.add_xpath('name', '//div[@class="product_name"]')
l.add_xpath('name', '//div[@class="product_title"]')
l.add_xpath('price', '//p[@id="price"]')
l.add_css('stock', 'p#stock')
l.add_value('last_updated', 'today') # you can also use literal values
l.add_xpath("name", '//div[@class="product_name"]')
l.add_xpath("name", '//div[@class="product_title"]')
l.add_xpath("price", '//p[@id="price"]')
l.add_css("stock", "p#stock")
l.add_value("last_updated", "today") # you can also use literal values
return l.load_item()
By quickly looking at that code, we can see the ``name`` field is being
@ -93,11 +96,14 @@ will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
:func:`~dataclasses.field` function, with a ``default`` argument:
.. code-block:: python
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
@ -122,14 +128,16 @@ processor). The result of the output processor is the final value that gets
assigned to the item.
Let's see an example to illustrate how the input and output processors are
called for a particular field (the same applies for any other field)::
called for a particular field (the same applies for any other field):
.. code-block:: python
l = ItemLoader(Product(), some_selector)
l.add_xpath('name', xpath1) # (1)
l.add_xpath('name', xpath2) # (2)
l.add_css('name', css) # (3)
l.add_value('name', 'test') # (4)
return l.load_item() # (5)
l.add_xpath("name", xpath1) # (1)
l.add_xpath("name", xpath2) # (2)
l.add_css("name", css) # (3)
l.add_value("name", "test") # (4)
return l.load_item() # (5)
So what happens is:
@ -184,13 +192,15 @@ processors <itemloaders:built-in-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared using a class definition syntax. Here is an example::
Item Loaders are declared using a class definition syntax. Here is an example:
.. code-block:: python
from itemloaders.processors import TakeFirst, MapCompose, Join
from scrapy.loader import ItemLoader
class ProductLoader(ItemLoader):
class ProductLoader(ItemLoader):
default_output_processor = TakeFirst()
name_in = MapCompose(str.title)
@ -215,16 +225,20 @@ As seen in the previous section, input and output processors can be declared in
the Item Loader definition, and it's very common to declare input processors
this way. However, there is one more place where you can specify the input and
output processors to use: in the :ref:`Item Field <topics-items-fields>`
metadata. Here is an example::
metadata. Here is an example:
.. code-block:: python
import scrapy
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
def filter_price(value):
if value.isdigit():
return value
class Product(scrapy.Item):
name = scrapy.Field(
input_processor=MapCompose(remove_tags),
@ -235,12 +249,15 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
.. code-block:: pycon
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
>>> il.add_value("price", ["&euro;", "<span>1000</span>"])
>>> il.load_item()
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:
@ -263,10 +280,12 @@ declaring, instantiating or using Item Loader. They are used to modify the
behaviour of the input/output processors.
For example, suppose you have a function ``parse_length`` which receives a text
value and extracts a length from it::
value and extracts a length from it:
.. code-block:: python
def parse_length(text, loader_context):
unit = loader_context.get('unit', 'm')
unit = loader_context.get("unit", "m")
# ... length parsing code goes here ...
return parsed_length
@ -278,22 +297,28 @@ function (``parse_length`` in this case) can thus use them.
There are several ways to modify Item Loader context values:
1. By modifying the currently active Item Loader context
(:attr:`~ItemLoader.context` attribute)::
(:attr:`~ItemLoader.context` attribute):
.. code-block:: python
loader = ItemLoader(product)
loader.context['unit'] = 'cm'
loader.context["unit"] = "cm"
2. On Item Loader instantiation (the keyword arguments of Item Loader
``__init__`` method are stored in the Item Loader context)::
``__init__`` method are stored in the Item Loader context):
loader = ItemLoader(product, unit='cm')
.. code-block:: python
loader = ItemLoader(product, unit="cm")
3. On Item Loader declaration, for those input/output processors that support
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
them::
them:
.. code-block:: python
class ProductLoader(ItemLoader):
length_out = MapCompose(parse_length, unit='cm')
length_out = MapCompose(parse_length, unit="cm")
ItemLoader objects
@ -323,25 +348,29 @@ Example::
Without nested loaders, you need to specify the full xpath (or css) for each value
that you wish to extract.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
loader.add_xpath('social', '//footer/a[@class = "social"]/@href')
loader.add_xpath('email', '//footer/a[@class = "email"]/@href')
loader.add_xpath("social", '//footer/a[@class = "social"]/@href')
loader.add_xpath("email", '//footer/a[@class = "email"]/@href')
loader.load_item()
Instead, you can create a nested loader with the footer selector and add values
relative to the footer. The functionality is the same but you avoid repeating
the footer selector.
Example::
Example:
.. code-block:: python
loader = ItemLoader(item=Item())
# load stuff not in the footer
footer_loader = loader.nested_xpath('//footer')
footer_loader.add_xpath('social', 'a[@class = "social"]/@href')
footer_loader.add_xpath('email', 'a[@class = "email"]/@href')
footer_loader = loader.nested_xpath("//footer")
footer_loader.add_xpath("social", 'a[@class = "social"]/@href')
footer_loader.add_xpath("email", 'a[@class = "email"]/@href')
# no need to call footer_loader.load_item()
loader.load_item()
@ -370,25 +399,32 @@ three dashes (e.g. ``---Plasma TV---``) and you don't want to end up scraping
those dashes in the final product names.
Here's how you can remove those dashes by reusing and extending the default
Product Item Loader (``ProductLoader``)::
Product Item Loader (``ProductLoader``):
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
def strip_dashes(x):
return x.strip('-')
return x.strip("-")
class SiteSpecificLoader(ProductLoader):
name_in = MapCompose(strip_dashes, ProductLoader.name_in)
Another case where extending Item Loaders can be very helpful is when you have
multiple source formats, for example XML and HTML. In the XML version you may
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
want to remove ``CDATA`` occurrences. Here's an example of how to do it:
.. code-block:: python
from itemloaders.processors import MapCompose
from myproject.ItemLoaders import ProductLoader
from myproject.utils.xml import remove_cdata
class XmlProductLoader(ProductLoader):
name_in = MapCompose(remove_cdata, ProductLoader.name_in)

111
docs/topics/logging.rst
View File

@ -39,16 +39,22 @@ How to log messages
===================
Here's a quick example of how to log a message using the ``logging.WARNING``
level::
level:
.. code-block:: python
import logging
logging.warning("This is a warning")
There are shortcuts for issuing log messages on any of the standard 5 levels,
and there's also a general ``logging.log`` method which takes a given level as
argument. If needed, the last example could be rewritten as::
argument. If needed, the last example could be rewritten as:
.. code-block:: python
import logging
logging.log(logging.WARNING, "This is a warning")
On top of that, you can create different "loggers" to encapsulate messages. (For
@ -59,24 +65,33 @@ constructions.
The previous examples use the root logger behind the scenes, which is a top level
logger where all messages are propagated to (unless otherwise specified). Using
``logging`` helpers is merely a shortcut for getting the root logger
explicitly, so this is also an equivalent of the last snippets::
explicitly, so this is also an equivalent of the last snippets:
.. code-block:: python
import logging
logger = logging.getLogger()
logger.warning("This is a warning")
You can use a different logger just by getting its name with the
``logging.getLogger`` function::
``logging.getLogger`` function:
.. code-block:: python
import logging
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
logger.warning("This is a warning")
Finally, you can ensure having a custom logger for any module you're working on
by using the ``__name__`` variable, which is populated with current module's
path::
path:
.. code-block:: python
import logging
logger = logging.getLogger(__name__)
logger.warning("This is a warning")
@ -94,33 +109,37 @@ Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
instance, which can be accessed and used like this::
instance, which can be accessed and used like this:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
class MySpider(scrapy.Spider):
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
self.logger.info("Parse function called on %s", response.url)
That logger is created using the Spider's name, but you can use any custom
Python logger you want. For example::
Python logger you want. For example:
.. code-block:: python
import logging
import scrapy
logger = logging.getLogger('mycustomlogger')
logger = logging.getLogger("mycustomlogger")
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['https://scrapy.org']
name = "myspider"
start_urls = ["https://scrapy.org"]
def parse(self, response):
logger.info('Parse function called on %s', response.url)
logger.info("Parse function called on %s", response.url)
.. _topics-logging-configuration:
@ -229,7 +248,9 @@ the crawl.
Next, we can see that the message has INFO level. To hide it
we should set logging level for ``scrapy.spidermiddlewares.httperror``
higher than INFO; next level after INFO is WARNING. It could be done
e.g. in the spider's ``__init__`` method::
e.g. in the spider's ``__init__`` method:
.. code-block:: python
import logging
import scrapy
@ -238,7 +259,7 @@ e.g. in the spider's ``__init__`` method::
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('scrapy.spidermiddlewares.httperror')
logger = logging.getLogger("scrapy.spidermiddlewares.httperror")
logger.setLevel(logging.WARNING)
super().__init__(*args, **kwargs)
@ -249,43 +270,53 @@ You can also filter log records by :class:`~logging.LogRecord` data. For
example, you can filter log records by message content using a substring or
a regular expression. Create a :class:`logging.Filter` subclass
and equip it with a regular expression pattern to
filter out unwanted messages::
filter out unwanted messages:
.. code-block:: python
import logging
import re
class ContentFilter(logging.Filter):
def filter(self, record):
match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
match = re.search(r"\d{3} [Ee]rror, retrying", record.message)
if match:
return False
A project-level filter may be attached to the root
handler created by Scrapy, this is a wieldy way to
filter all loggers in different parts of the project
(middlewares, spider, etc.)::
(middlewares, spider, etc.):
import logging
import scrapy
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
for handler in logging.root.handlers:
handler.addFilter(ContentFilter())
Alternatively, you may choose a specific logger
and hide it without affecting other loggers::
and hide it without affecting other loggers:
.. code-block:: python
import logging
import scrapy
class MySpider(scrapy.Spider):
# ...
def __init__(self, *args, **kwargs):
logger = logging.getLogger('my_logger')
logger = logging.getLogger("my_logger")
logger.addFilter(ContentFilter())
scrapy.utils.log module
=======================
@ -306,14 +337,14 @@ scrapy.utils.log module
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
This is an example on how to redirect ``INFO`` or higher messages to a file:
.. code-block:: python
import logging
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
level=logging.INFO
filename="log.txt", format="%(levelname)s: %(message)s", level=logging.INFO
)
Refer to :ref:`run-from-script` for more details about using Scrapy this

177
docs/topics/media-pipeline.rst
View File

@ -87,13 +87,17 @@ Enabling your Media Pipeline
To enable your media pipeline you must first add it to your project
:setting:`ITEM_PIPELINES` setting.
For Images Pipeline, use::
For Images Pipeline, use:
ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}
.. code-block:: python
For Files Pipeline, use::
ITEM_PIPELINES = {"scrapy.pipelines.images.ImagesPipeline": 1}
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
For Files Pipeline, use:
.. code-block:: python
ITEM_PIPELINES = {"scrapy.pipelines.files.FilesPipeline": 1}
.. note::
You can also use both the Files and Images Pipeline at the same time.
@ -103,13 +107,17 @@ Then, configure the target storage setting to a valid value that will be used
for storing the downloaded images. Otherwise the pipeline will remain disabled,
even if you include it in the :setting:`ITEM_PIPELINES` setting.
For the Files Pipeline, set the :setting:`FILES_STORE` setting::
For the Files Pipeline, set the :setting:`FILES_STORE` setting:
FILES_STORE = '/path/to/valid/dir'
.. code-block:: python
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
FILES_STORE = "/path/to/valid/dir"
IMAGES_STORE = '/path/to/valid/dir'
For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:
.. code-block:: python
IMAGES_STORE = "/path/to/valid/dir"
.. _topics-file-naming:
@ -157,10 +165,11 @@ By overriding ``file_path`` like this:
import hashlib
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
image_perspective = request.url.split('/')[-2]
image_filename = f'{image_url_hash}_{image_perspective}.jpg'
image_perspective = request.url.split("/")[-2]
image_filename = f"{image_url_hash}_{image_perspective}.jpg"
return image_filename
@ -233,30 +242,38 @@ If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.
For example, this is a valid :setting:`IMAGES_STORE` value::
For example, this is a valid :setting:`IMAGES_STORE` value:
IMAGES_STORE = 's3://bucket/images'
.. code-block:: python
IMAGES_STORE = "s3://bucket/images"
You can modify the Access Control List (ACL) policy used for the stored files,
which is defined by the :setting:`FILES_STORE_S3_ACL` and
:setting:`IMAGES_STORE_S3_ACL` settings. By default, the ACL is set to
``private``. To make the files publicly available use the ``public-read``
policy::
policy:
IMAGES_STORE_S3_ACL = 'public-read'
.. code-block:: python
IMAGES_STORE_S3_ACL = "public-read"
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
settings::
settings:
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
.. code-block:: python
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
AWS_ENDPOINT_URL = "http://minio.example.com:9000"
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:
.. code-block:: python
AWS_USE_SSL = False # or True (None by default)
AWS_VERIFY = False # or True (None by default)
.. _botocore: https://github.com/boto/botocore
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
@ -277,10 +294,12 @@ bucket. Scrapy will automatically upload the files to the bucket. (requires `goo
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
IMAGES_STORE = 'gs://bucket/images/'
GCS_PROJECT_ID = 'project_id'
.. code-block:: python
IMAGES_STORE = "gs://bucket/images/"
GCS_PROJECT_ID = "project_id"
For information about authentication, see this `documentation`_.
@ -291,9 +310,11 @@ which is defined by the :setting:`FILES_STORE_GCS_ACL` and
:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
To make the files publicly available use the ``publicRead``
policy::
policy:
IMAGES_STORE_GCS_ACL = 'publicRead'
.. code-block:: python
IMAGES_STORE_GCS_ACL = "publicRead"
For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
@ -318,10 +339,13 @@ respectively), the pipeline will put the results under the respective field
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.Item` class::
``images`` field. For instance, using the :class:`~scrapy.Item` class:
.. code-block:: python
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
@ -331,16 +355,20 @@ If you want to use another field name for the URLs key or for the results key,
it is also possible to override it.
For the Files Pipeline, set :setting:`FILES_URLS_FIELD` and/or
:setting:`FILES_RESULT_FIELD` settings::
:setting:`FILES_RESULT_FIELD` settings:
FILES_URLS_FIELD = 'field_name_for_your_files_urls'
FILES_RESULT_FIELD = 'field_name_for_your_processed_files'
.. code-block:: python
FILES_URLS_FIELD = "field_name_for_your_files_urls"
FILES_RESULT_FIELD = "field_name_for_your_processed_files"
For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
:setting:`IMAGES_RESULT_FIELD` settings::
:setting:`IMAGES_RESULT_FIELD` settings:
IMAGES_URLS_FIELD = 'field_name_for_your_images_urls'
IMAGES_RESULT_FIELD = 'field_name_for_your_processed_images'
.. code-block:: python
IMAGES_URLS_FIELD = "field_name_for_your_images_urls"
IMAGES_RESULT_FIELD = "field_name_for_your_processed_images"
If you need something more complex and want to override the custom pipeline
behaviour, see :ref:`topics-media-pipeline-override`.
@ -366,7 +394,9 @@ File expiration
The Image Pipeline avoids downloading files that were downloaded recently. To
adjust this retention delay use the :setting:`FILES_EXPIRES` setting (or
:setting:`IMAGES_EXPIRES`, in case of Images Pipeline), which
specifies the delay in number of days::
specifies the delay in number of days:
.. code-block:: python
# 120 days of delay for files expiration
FILES_EXPIRES = 120
@ -400,11 +430,13 @@ images.
In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
where the keys are the thumbnail names and the values are their dimensions.
For example::
For example:
.. code-block:: python
IMAGES_THUMBS = {
'small': (50, 50),
'big': (270, 270),
"small": (50, 50),
"big": (270, 270),
}
When you use this feature, the Images Pipeline will create thumbnails of the
@ -495,17 +527,19 @@ See here the methods that you can override in your custom Files Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
class MyFilesPipeline(FilesPipeline):
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -521,13 +555,16 @@ See here the methods that you can override in your custom Files Pipeline:
As seen on the workflow, the pipeline will get the URLs of the images to
download from the item. In order to do this, you can override the
:meth:`~get_media_requests` method and return a Request for each
file URL::
file URL:
.. code-block:: python
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
for file_url in adapter["file_urls"]:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -567,15 +604,22 @@ See here the methods that you can override in your custom Files Pipeline:
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
Here's a typical value of the ``results`` argument::
Here's a typical value of the ``results`` argument:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
'url': 'http://www.example.com/files/product1.pdf',
'status': 'downloaded'}),
(False,
Failure(...))]
.. code-block:: python
[
(
True,
{
"checksum": "2b00042f7481c7b056c4b410d28f33cf",
"path": "full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg",
"url": "http://www.example.com/files/product1.pdf",
"status": "downloaded",
},
),
(False, Failure(...)),
]
By default the :meth:`get_media_requests` method returns ``None`` which
means there are no files to download for the item.
@ -592,17 +636,20 @@ See here the methods that you can override in your custom Files Pipeline:
Here is an example of the :meth:`~item_completed` method where we
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
item field, and we drop the item if it doesn't contain any files:
.. code-block:: python
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
file_paths = [x["path"] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
adapter["file_paths"] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -634,17 +681,19 @@ See here the methods that you can override in your custom Images Pipeline:
For example, if file URLs end like regular paths (e.g.
``https://example.com/a/b/c/foo.png``), you can use the following
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
original filenames (e.g. ``files/foo.png``):
.. code-block:: python
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + PurePosixPath(urlparse(request.url).path).name
return "files/" + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -700,33 +749,35 @@ Custom Images pipeline example
==============================
Here is a full example of the Images Pipeline whose methods are exemplified
above::
above:
.. code-block:: python
import scrapy
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
class MyImagesPipeline(ImagesPipeline):
def get_media_requests(self, item, info):
for image_url in item['image_urls']:
for image_url in item["image_urls"]:
yield scrapy.Request(image_url)
def item_completed(self, results, item, info):
image_paths = [x['path'] for ok, x in results if ok]
image_paths = [x["path"] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
adapter["image_paths"] = image_paths
return item
To enable your custom media pipeline component you must add its class import path to the
:setting:`ITEM_PIPELINES` setting, like in the following example::
:setting:`ITEM_PIPELINES` setting, like in the following example:
ITEM_PIPELINES = {
'myproject.pipelines.MyImagesPipeline': 300
}
.. code-block:: python
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5

55
docs/topics/practices.rst
View File

@ -25,23 +25,27 @@ the one used by all Scrapy commands.
Here's an example showing how to run a single spider with it.
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
class MySpider(scrapy.Spider):
# Your spider definition
...
process = CrawlerProcess(settings={
"FEEDS": {
"items.json": {"format": "json"},
},
})
process = CrawlerProcess(
settings={
"FEEDS": {
"items.json": {"format": "json"},
},
}
)
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
process.start() # the script will block here until the crawling is finished
Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
documentation to get acquainted with its usage details.
@ -55,7 +59,7 @@ instance with your project settings.
What follows is a working example of how to do that, using the `testspiders`_
project as example.
::
.. code-block:: python
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
@ -63,8 +67,8 @@ project as example.
process = CrawlerProcess(get_project_settings())
# 'followall' is the name of one of the spiders of the project.
process.crawl('followall', domain='scrapy.org')
process.start() # the script will block here until the crawling is finished
process.crawl("followall", domain="scrapy.org")
process.start() # the script will block here until the crawling is finished
There's another Scrapy utility that provides more control over the crawling
process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper
@ -84,23 +88,25 @@ returned by the :meth:`CrawlerRunner.crawl
Here's an example of its usage, along with a callback to manually stop the
reactor after ``MySpider`` has finished running.
::
.. code-block:: python
from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class MySpider(scrapy.Spider):
# Your spider definition
...
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
runner = CrawlerRunner()
d = runner.crawl(MySpider)
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
reactor.run() # the script will block here until the crawling is finished
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
@ -115,29 +121,32 @@ the :ref:`internal API <topics-api>`.
Here is an example that runs multiple spiders simultaneously:
::
.. code-block:: python
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
process = CrawlerProcess(settings)
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start() # the script will block here until all crawling jobs are finished
process.start() # the script will block here until all crawling jobs are finished
Same example using :class:`~scrapy.crawler.CrawlerRunner`:
::
.. code-block:: python
import scrapy
from twisted.internet import reactor
@ -145,14 +154,17 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
configure_logging()
settings = get_project_settings()
runner = CrawlerRunner(settings)
@ -161,37 +173,42 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
d = runner.join()
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until all crawling jobs are finished
reactor.run() # the script will block here until all crawling jobs are finished
Same example but running the spiders sequentially by chaining the deferreds:
::
.. code-block:: python
from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
settings = get_project_settings()
configure_logging(settings)
runner = CrawlerRunner(settings)
@defer.inlineCallbacks
def crawl():
yield runner.crawl(MySpider1)
yield runner.crawl(MySpider2)
reactor.stop()
crawl()
reactor.run() # the script will block here until the last crawl call is finished
reactor.run() # the script will block here until the last crawl call is finished
Different spiders can set different values for the same setting, but when they
run in the same process it may be impossible, by design or because of some

217
docs/topics/request-response.rst
View File

@ -32,11 +32,20 @@ Request objects
:type url: str
:param callback: the function that will be called with the response of this
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.Spider.parse` method will be used.
Note that if exceptions are raised during processing, errback is called instead.
request (once it's downloaded) as its first parameter.
In addition to a function, the following values are supported:
- ``None`` (default), which indicates that the spider's
:meth:`~scrapy.Spider.parse` method must be used.
- :func:`~scrapy.http.request.NO_CALLBACK`
For more information, see
:ref:`topics-request-response-ref-request-callback-arguments`.
.. note:: If exceptions are raised during processing, ``errback`` is
called instead.
:type callback: collections.abc.Callable
@ -67,18 +76,30 @@ Request objects
:param cookies: the request cookies. These can be sent in two forms.
1. Using a dict::
1. Using a dict:
request_with_cookies = Request(url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'})
.. code-block:: python
2. Using a list of dicts::
request_with_cookies = Request(
url="http://www.example.com",
cookies={"currency": "USD", "country": "UY"},
)
request_with_cookies = Request(url="http://www.example.com",
cookies=[{'name': 'currency',
'value': 'USD',
'domain': 'example.com',
'path': '/currency'}])
2. Using a list of dicts:
.. code-block:: python
request_with_cookies = Request(
url="http://www.example.com",
cookies=[
{
"name": "currency",
"value": "USD",
"domain": "example.com",
"path": "/currency",
},
],
)
The latter form allows for customizing the ``domain`` and ``path``
attributes of the cookie. This is only useful if the cookies are saved
@ -95,12 +116,14 @@ Request objects
in :attr:`request.meta <scrapy.Request.meta>`.
Example of a request that sends manually-defined cookies and ignores
cookie storage::
cookie storage:
.. code-block:: python
Request(
url="http://www.example.com",
cookies={'currency': 'USD', 'country': 'UY'},
meta={'dont_merge_cookies': True},
cookies={"currency": "USD", "country": "UY"},
meta={"dont_merge_cookies": True},
)
For more info see :ref:`cookies-mw`.
@ -228,6 +251,8 @@ Request objects
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.http.request.NO_CALLBACK
.. autofunction:: scrapy.utils.request.request_from_dict
@ -240,11 +265,15 @@ The callback of a request is a function that will be called when the response
of that request is downloaded. The callback function will be called with the
downloaded :class:`Response` object as its first argument.
Example::
Example:
.. code-block:: python
def parse_page1(self, response):
return scrapy.Request("http://www.example.com/some_page.html",
callback=self.parse_page2)
return scrapy.Request(
"http://www.example.com/some_page.html", callback=self.parse_page2
)
def parse_page2(self, response):
# this would log http://www.example.com/some_page.html
@ -255,15 +284,18 @@ functions so you can receive the arguments later, in the second callback.
The following example shows how to achieve this by using the
:attr:`Request.cb_kwargs` attribute:
::
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url))
request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
cb_kwargs=dict(main_url=response.url),
)
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
yield request
def parse_page2(self, response, main_url, foo):
yield dict(
main_url=main_url,
@ -289,7 +321,9 @@ It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
errors if needed:
.. code-block:: python
import scrapy
@ -297,24 +331,28 @@ errors if needed::
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
class ErrbackSpider(scrapy.Spider):
name = "errback_example"
start_urls = [
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
"http://www.httpbin.org/", # HTTP 200 expected
"http://www.httpbin.org/status/404", # Not found error
"http://www.httpbin.org/status/500", # server issue
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
"https://example.invalid/", # DNS error expected
]
def start_requests(self):
for u in self.start_urls:
yield scrapy.Request(u, callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True)
yield scrapy.Request(
u,
callback=self.parse_httpbin,
errback=self.errback_httpbin,
dont_filter=True,
)
def parse_httpbin(self, response):
self.logger.info('Got successful response from {}'.format(response.url))
self.logger.info("Got successful response from {}".format(response.url))
# do something useful here...
def errback_httpbin(self, failure):
@ -328,16 +366,16 @@ errors if needed::
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error('HttpError on %s', response.url)
self.logger.error("HttpError on %s", response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error('DNSLookupError on %s', request.url)
self.logger.error("DNSLookupError on %s", request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
self.logger.error("TimeoutError on %s", request.url)
.. _errback-cb_kwargs:
@ -348,21 +386,27 @@ Accessing additional data in errback functions
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
achieve this by using ``Failure.request.cb_kwargs``:
.. code-block:: python
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
request = scrapy.Request(
"http://www.example.com/index.html",
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url),
)
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
main_url=failure.request.cb_kwargs["main_url"],
)
@ -509,18 +553,20 @@ in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
For example, to take the value of a request header named ``X-ID`` into
account::
account:
.. code-block:: python
# my_project/settings.py
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
REQUEST_FINGERPRINTER_CLASS = "my_project.utils.RequestFingerprinter"
# my_project/utils.py
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
class RequestFingerprinter:
def fingerprint(self, request):
return fingerprint(request, include_headers=['X-ID'])
return fingerprint(request, include_headers=["X-ID"])
You can also write your own fingerprinting logic from scratch.
@ -536,15 +582,17 @@ you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
references to them in your cache dictionary.
For example, to take into account only the URL of a request, without any prior
URL canonicalization or taking the request method or body into account::
URL canonicalization or taking the request method or body into account:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
from scrapy.utils.python import to_bytes
class RequestFingerprinter:
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
@ -558,21 +606,25 @@ If you need to be able to override the request fingerprinting for arbitrary
requests from your spider callbacks, you may implement a request fingerprinter
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
when available, and then falls back to
:func:`scrapy.utils.request.fingerprint`. For example::
:func:`scrapy.utils.request.fingerprint`. For example:
.. code-block:: python
from scrapy.utils.request import fingerprint
class RequestFingerprinter:
class RequestFingerprinter:
def fingerprint(self, request):
if 'fingerprint' in request.meta:
return request.meta['fingerprint']
if "fingerprint" in request.meta:
return request.meta["fingerprint"]
return fingerprint(request)
If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
without using the deprecated ``'2.6'`` value of the
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
request fingerprinter::
request fingerprinter:
.. code-block:: python
from hashlib import sha1
from weakref import WeakKeyDictionary
@ -580,8 +632,8 @@ request fingerprinter::
from scrapy.utils.python import to_bytes
from w3lib.url import canonicalize_url
class RequestFingerprinter:
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
@ -589,7 +641,7 @@ request fingerprinter::
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b'')
fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
@ -718,7 +770,9 @@ Stopping the download of a Response
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
signals will stop the download of a given response. See the following example::
signals will stop the download of a given response. See the following example:
.. code-block:: python
import scrapy
@ -730,7 +784,9 @@ signals will stop the download of a given response. See the following example::
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
crawler.signals.connect(
spider.on_bytes_received, signal=scrapy.signals.bytes_received
)
return spider
def parse(self, response):
@ -859,11 +915,17 @@ Using FormRequest to send data via HTTP POST
If you want to simulate a HTML Form POST in your spider and send a couple of
key-value fields, you can return a :class:`FormRequest` object (from your
spider) like this::
spider) like this:
return [FormRequest(url="http://www.example.com/post/action",
formdata={'name': 'John Doe', 'age': '27'},
callback=self.after_post)]
.. code-block:: python
return [
FormRequest(
url="http://www.example.com/post/action",
formdata={"name": "John Doe", "age": "27"},
callback=self.after_post,
)
]
.. _topics-request-response-ref-request-userlogin:
@ -875,25 +937,28 @@ type="hidden">`` elements, such as session related data or authentication
tokens (for login pages). When scraping, you'll want these fields to be
automatically pre-populated and only override a couple of them, such as the
user name and password. You can use the :meth:`FormRequest.from_response`
method for this job. Here's an example spider which uses it::
method for this job. Here's an example spider which uses it:
.. code-block:: python
import scrapy
def authentication_failed(response):
# TODO: Check the contents of the response and return True if it failed
# or False if it succeeded.
pass
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
name = "example.com"
start_urls = ["http://www.example.com/users/login.php"]
def parse(self, response):
return scrapy.FormRequest.from_response(
response,
formdata={'username': 'john', 'password': 'secret'},
callback=self.after_login
formdata={"username": "john", "password": "secret"},
callback=self.after_login,
)
def after_login(self, response):
@ -933,13 +998,15 @@ dealing with JSON requests.
JsonRequest usage example
-------------------------
Sending a JSON POST request with a JSON payload::
Sending a JSON POST request with a JSON payload:
.. code-block:: python
data = {
'name1': 'value1',
'name2': 'value2',
"name1": "value1",
"name2": "value2",
}
yield JsonRequest(url='http://www.example.com/post/action', data=data)
yield JsonRequest(url="http://www.example.com/post/action", data=data)
Response objects
@ -1174,8 +1241,10 @@ TextResponse objects
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. code-block:: pycon
>>> str(b"body")
"b'body'"
.. attribute:: TextResponse.encoding

619
docs/topics/selectors.rst
View File

@ -51,16 +51,20 @@ Constructing selectors
Response objects expose a :class:`~scrapy.Selector` instance
on ``.selector`` attribute:
>>> response.selector.xpath('//span/text()').get()
'good'
.. code-block:: pycon
>>> response.selector.xpath("//span/text()").get()
'good'
Querying responses using XPath and CSS is so common that responses include two
more shortcuts: ``response.xpath()`` and ``response.css()``:
>>> response.xpath('//span/text()').get()
'good'
>>> response.css('span::text').get()
'good'
.. code-block:: pycon
>>> response.xpath("//span/text()").get()
'good'
>>> response.css("span::text").get()
'good'
Scrapy selectors are instances of :class:`~scrapy.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
@ -75,19 +79,23 @@ you can also ensure the response body is parsed only once.
But if required, it is possible to use ``Selector`` directly.
Constructing from text:
>>> from scrapy.selector import Selector
>>> body = '<html><body><span>good</span></body></html>'
>>> Selector(text=body).xpath('//span/text()').get()
'good'
.. code-block:: pycon
>>> from scrapy.selector import Selector
>>> body = "<html><body><span>good</span></body></html>"
>>> Selector(text=body).xpath("//span/text()").get()
'good'
Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
:class:`~scrapy.http.TextResponse` subclasses:
>>> from scrapy.selector import Selector
>>> from scrapy.http import HtmlResponse
>>> response = HtmlResponse(url='http://example.com', body=body)
>>> Selector(response=response).xpath('//span/text()').get()
'good'
.. code-block:: pycon
>>> from scrapy.selector import Selector
>>> from scrapy.http import HtmlResponse
>>> response = HtmlResponse(url="http://example.com", body=body)
>>> Selector(response=response).xpath("//span/text()").get()
'good'
``Selector`` automatically chooses the best parsing rules
(XML vs HTML) based on input type.
@ -124,16 +132,20 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
page, let's construct an XPath for selecting the text inside the title tag:
>>> response.xpath('//title/text()')
[<Selector xpath='//title/text()' data='Example website'>]
.. code-block:: pycon
>>> response.xpath("//title/text()")
[<Selector xpath='//title/text()' data='Example website'>]
To actually extract the textual data, you must call the selector ``.get()``
or ``.getall()`` methods, as follows:
>>> response.xpath('//title/text()').getall()
['Example website']
>>> response.xpath('//title/text()').get()
'Example website'
.. code-block:: pycon
>>> response.xpath("//title/text()").getall()
['Example website']
>>> response.xpath("//title/text()").get()
'Example website'
``.get()`` always returns a single result; if there are several matches,
content of a first match is returned; if there are no matches, None
@ -142,98 +154,116 @@ is returned. ``.getall()`` returns a list with all results.
Notice that CSS selectors can select text or attribute nodes using CSS3
pseudo-elements:
>>> response.css('title::text').get()
'Example website'
.. code-block:: pycon
>>> response.css("title::text").get()
'Example website'
As you can see, ``.xpath()`` and ``.css()`` methods return a
:class:`~scrapy.selector.SelectorList` instance, which is a list of new
selectors. This API can be used for quickly selecting nested data:
>>> response.css('img').xpath('@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
.. code-block:: pycon
>>> response.css("img").xpath("@src").getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
If you want to extract only the first matched element, you can call the
selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
previous Scrapy versions):
>>> response.xpath('//div[@id="images"]/a/text()').get()
'Name: My image 1 '
.. code-block:: pycon
>>> response.xpath('//div[@id="images"]/a/text()').get()
'Name: My image 1 '
It returns ``None`` if no element was found:
>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
True
.. code-block:: pycon
>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
True
A default return value can be provided as an argument, to be used instead
of ``None``:
>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
'not-found'
.. code-block:: pycon
>>> response.xpath('//div[@id="not-exists"]/text()').get(default="not-found")
'not-found'
Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
using ``.attrib`` property of a :class:`~scrapy.Selector`:
>>> [img.attrib['src'] for img in response.css('img')]
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
.. code-block:: pycon
>>> [img.attrib["src"] for img in response.css("img")]
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
As a shortcut, ``.attrib`` is also available on SelectorList directly;
it returns attributes for the first matching element:
>>> response.css('img').attrib['src']
'image1_thumb.jpg'
.. code-block:: pycon
>>> response.css("img").attrib["src"]
'image1_thumb.jpg'
This is most useful when only a single result is expected, e.g. when selecting
by id, or selecting unique elements on a web page:
>>> response.css('base').attrib['href']
'http://example.com/'
.. code-block:: pycon
>>> response.css("base").attrib["href"]
'http://example.com/'
Now we're going to get the base URL and some image links:
>>> response.xpath('//base/@href').get()
'http://example.com/'
.. code-block:: pycon
>>> response.css('base::attr(href)').get()
'http://example.com/'
>>> response.xpath("//base/@href").get()
'http://example.com/'
>>> response.css('base').attrib['href']
'http://example.com/'
>>> response.css("base::attr(href)").get()
'http://example.com/'
>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.css("base").attrib["href"]
'http://example.com/'
>>> response.css('a[href*=image]::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.css("a[href*=image]::attr(href)").getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
>>> response.css('a[href*=image] img::attr(src)').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
>>> response.css("a[href*=image] img::attr(src)").getall()
['image1_thumb.jpg',
'image2_thumb.jpg',
'image3_thumb.jpg',
'image4_thumb.jpg',
'image5_thumb.jpg']
.. _topics-selectors-css-extensions:
@ -260,45 +290,55 @@ Examples:
* ``title::text`` selects children text nodes of a descendant ``<title>`` element:
>>> response.css('title::text').get()
'Example website'
.. code-block:: pycon
>>> response.css("title::text").get()
'Example website'
* ``*::text`` selects all descendant text nodes of the current selector context:
>>> response.css('#images *::text').getall()
['\n ',
'Name: My image 1 ',
'\n ',
'Name: My image 2 ',
'\n ',
'Name: My image 3 ',
'\n ',
'Name: My image 4 ',
'\n ',
'Name: My image 5 ',
'\n ']
.. code-block:: pycon
>>> response.css("#images *::text").getall()
['\n ',
'Name: My image 1 ',
'\n ',
'Name: My image 2 ',
'\n ',
'Name: My image 3 ',
'\n ',
'Name: My image 4 ',
'\n ',
'Name: My image 5 ',
'\n ']
* ``foo::text`` returns no results if ``foo`` element exists, but contains
no text (i.e. text is empty):
>>> response.css('img::text').getall()
[]
.. code-block:: pycon
>>> response.css("img::text").getall()
[]
This means ``.css('foo::text').get()`` could return None even if an element
exists. Use ``default=''`` if you always want a string:
>>> response.css('img::text').get()
>>> response.css('img::text').get(default='')
''
.. code-block:: pycon
>>> response.css("img::text").get()
>>> response.css("img::text").get(default="")
''
* ``a::attr(href)`` selects the *href* attribute value of descendant links:
>>> response.css('a::attr(href)').getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
.. code-block:: pycon
>>> response.css("a::attr(href)").getall()
['image1.html',
'image2.html',
'image3.html',
'image4.html',
'image5.html']
.. note::
See also: :ref:`selecting-attributes`.
@ -319,23 +359,26 @@ The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
of the same type, so you can call the selection methods for those selectors
too. Here's an example:
>>> links = response.xpath('//a[contains(@href, "image")]')
>>> links.getall()
['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
.. code-block:: pycon
>>> for index, link in enumerate(links):
... href_xpath = link.xpath('@href').get()
... img_xpath = link.xpath('img/@src').get()
... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}')
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
>>> links = response.xpath('//a[contains(@href, "image")]')
>>> links.getall()
['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
>>> for index, link in enumerate(links):
... href_xpath = link.xpath("@href").get()
... img_xpath = link.xpath("img/@src").get()
... print(f"Link number {index} points to url {href_xpath!r} and image {img_xpath!r}")
...
Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
.. _selecting-attributes:
@ -345,8 +388,10 @@ Selecting element attributes
There are several ways to get a value of an attribute. First, one can use
XPath syntax:
>>> response.xpath("//a/@href").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
.. code-block:: pycon
>>> response.xpath("//a/@href").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
XPath syntax has a few advantages: it is a standard XPath feature, and
``@attributes`` can be used in other parts of an XPath expression - e.g.
@ -355,30 +400,38 @@ it is possible to filter by attribute value.
Scrapy also provides an extension to CSS selectors (``::attr(...)``)
which allows to get attribute values:
>>> response.css('a::attr(href)').getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
.. code-block:: pycon
>>> response.css("a::attr(href)").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
In addition to that, there is a ``.attrib`` property of Selector.
You can use it if you prefer to lookup attributes in Python
code, without using XPaths or CSS extensions:
>>> [a.attrib['href'] for a in response.css('a')]
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
.. code-block:: pycon
>>> [a.attrib["href"] for a in response.css("a")]
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
This property is also available on SelectorList; it returns a dictionary
with attributes of a first matching element. It is convenient to use when
a selector is expected to give a single result (e.g. when selecting by element
ID, or when selecting an unique element on a page):
>>> response.css('base').attrib
{'href': 'http://example.com/'}
>>> response.css('base').attrib['href']
'http://example.com/'
.. code-block:: pycon
>>> response.css("base").attrib
{'href': 'http://example.com/'}
>>> response.css("base").attrib["href"]
'http://example.com/'
``.attrib`` property of an empty SelectorList is empty:
>>> response.css('foo').attrib
{}
.. code-block:: pycon
>>> response.css("foo").attrib
{}
Using selectors with regular expressions
----------------------------------------
@ -391,19 +444,23 @@ can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
<topics-selectors-htmlcode>` above:
>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
['My image 1',
'My image 2',
'My image 3',
'My image 4',
'My image 5']
.. code-block:: pycon
>>> response.xpath('//a[contains(@href, "image")]/text()').re(r"Name:\s*(.*)")
['My image 1',
'My image 2',
'My image 3',
'My image 4',
'My image 5']
There's an additional helper reciprocating ``.get()`` (and its
alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
Use it to extract just the first matching string:
>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
'My image 1'
.. code-block:: pycon
>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r"Name:\s*(.*)")
'My image 1'
.. _old-extraction-api:
@ -423,28 +480,36 @@ The following examples show how these methods map to each other.
1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:
>>> response.css('a::attr(href)').get()
.. code-block:: pycon
>>> response.css("a::attr(href)").get()
'image1.html'
>>> response.css('a::attr(href)').extract_first()
>>> response.css("a::attr(href)").extract_first()
'image1.html'
2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:
>>> response.css('a::attr(href)').getall()
.. code-block:: pycon
>>> response.css("a::attr(href)").getall()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
>>> response.css('a::attr(href)').extract()
>>> response.css("a::attr(href)").extract()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
3. ``Selector.get()`` is the same as ``Selector.extract()``:
>>> response.css('a::attr(href)')[0].get()
.. code-block:: pycon
>>> response.css("a::attr(href)")[0].get()
'image1.html'
>>> response.css('a::attr(href)')[0].extract()
>>> response.css("a::attr(href)")[0].extract()
'image1.html'
4. For consistency, there is also ``Selector.getall()``, which returns a list:
>>> response.css('a::attr(href)')[0].getall()
.. code-block:: pycon
>>> response.css("a::attr(href)")[0].getall()
['image1.html']
So, the main difference is that output of ``.get()`` and ``.getall()`` methods
@ -482,24 +547,35 @@ with ``/``, that XPath will be absolute to the document and not relative to the
For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
elements. First, you would get all ``<div>`` elements:
>>> divs = response.xpath('//div')
.. code-block:: pycon
>>> divs = response.xpath("//div")
At first, you may be tempted to use the following approach, which is wrong, as
it actually extracts all ``<p>`` elements from the document, not only those
inside ``<div>`` elements:
>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
... print(p.get())
.. code-block:: pycon
>>> for p in divs.xpath("//p"): # this is wrong - gets all <p> from the whole document
... print(p.get())
...
This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):
>>> for p in divs.xpath('.//p'): # extracts all <p> inside
... print(p.get())
.. code-block:: pycon
>>> for p in divs.xpath(".//p"): # extracts all <p> inside
... print(p.get())
...
Another common case would be to extract all direct ``<p>`` children:
>>> for p in divs.xpath('p'):
... print(p.get())
.. code-block:: pycon
>>> for p in divs.xpath("p"):
... print(p.get())
...
For more details about relative XPaths see the `Location Paths`_ section in the
XPath specification.
@ -522,10 +598,14 @@ class name that shares the string ``someclass``.
As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
you can just select by class using CSS and then switch to XPath when needed:
>>> from scrapy import Selector
>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
>>> sel.css('.shout').xpath('./time/@datetime').getall()
['2014-07-23 19:00']
.. code-block:: pycon
>>> from scrapy import Selector
>>> sel = Selector(
... text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>'
... )
>>> sel.css(".shout").xpath("./time/@datetime").getall()
['2014-07-23 19:00']
This is cleaner than using the verbose XPath trick shown above. Just remember
to use the ``.`` in the XPath expressions that will follow.
@ -539,39 +619,51 @@ Beware of the difference between //node[1] and (//node)[1]
Example:
>>> from scrapy import Selector
>>> sel = Selector(text="""
....: <ul class="list">
....: <li>1</li>
....: <li>2</li>
....: <li>3</li>
....: </ul>
....: <ul class="list">
....: <li>4</li>
....: <li>5</li>
....: <li>6</li>
....: </ul>""")
>>> xp = lambda x: sel.xpath(x).getall()
.. code-block:: pycon
>>> from scrapy import Selector
>>> sel = Selector(
... text="""
... <ul class="list">
... <li>1</li>
... <li>2</li>
... <li>3</li>
... </ul>
... <ul class="list">
... <li>4</li>
... <li>5</li>
... <li>6</li>
... </ul>"""
... )
>>> xp = lambda x: sel.xpath(x).getall()
This gets all first ``<li>`` elements under whatever it is its parent:
>>> xp("//li[1]")
['<li>1</li>', '<li>4</li>']
.. code-block:: pycon
>>> xp("//li[1]")
['<li>1</li>', '<li>4</li>']
And this gets the first ``<li>`` element in the whole document:
>>> xp("(//li)[1]")
['<li>1</li>']
.. code-block:: pycon
>>> xp("(//li)[1]")
['<li>1</li>']
This gets all first ``<li>`` elements under an ``<ul>`` parent:
>>> xp("//ul/li[1]")
['<li>1</li>', '<li>4</li>']
.. code-block:: pycon
>>> xp("//ul/li[1]")
['<li>1</li>', '<li>4</li>']
And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:
>>> xp("(//ul/li)[1]")
['<li>1</li>']
.. code-block:: pycon
>>> xp("(//ul/li)[1]")
['<li>1</li>']
Using text nodes in a condition
-------------------------------
@ -585,32 +677,44 @@ a string function like ``contains()`` or ``starts-with()``, it results in the te
Example:
>>> from scrapy import Selector
>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
.. code-block:: pycon
>>> from scrapy import Selector
>>> sel = Selector(
... text='<a href="#">Click here to go to the <strong>Next Page</strong></a>'
... )
Converting a *node-set* to string:
>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
['Click here to go to the ', 'Next Page']
>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
['Click here to go to the ']
.. code-block:: pycon
>>> sel.xpath("//a//text()").getall() # take a peek at the node-set
['Click here to go to the ', 'Next Page']
>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
['Click here to go to the ']
A *node* converted to a string, however, puts together the text of itself plus of all its descendants:
>>> sel.xpath("//a[1]").getall() # select the first node
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
>>> sel.xpath("string(//a[1])").getall() # convert it to string
['Click here to go to the Next Page']
.. code-block:: pycon
>>> sel.xpath("//a[1]").getall() # select the first node
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
>>> sel.xpath("string(//a[1])").getall() # convert it to string
['Click here to go to the Next Page']
So, using the ``.//text()`` node-set won't select anything in this case:
>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
[]
.. code-block:: pycon
>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
[]
But using the ``.`` to mean the node, works:
>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
.. code-block:: pycon
>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions
@ -628,15 +732,19 @@ which are then substituted with values passed with the query.
Here's an example to match an element based on its "id" attribute value,
without hard-coding it (that was shown previously):
>>> # `$val` used in the expression, a `val` argument needs to be passed
>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
'Name: My image 1 '
.. code-block:: pycon
>>> # `$val` used in the expression, a `val` argument needs to be passed
>>> response.xpath("//div[@id=$val]/a/text()", val="images").get()
'Name: My image 1 '
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
five ``<a>`` children (here we pass the value ``5`` as an integer):
>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
'images'
.. code-block:: pycon
>>> response.xpath("//div[count(a)=$cnt]/@id", cnt=5).get()
'images'
All variable references must have a binding value when calling ``.xpath()``
(otherwise you'll get a ``ValueError: XPath error:`` exception).
@ -688,17 +796,21 @@ You can see several namespace declarations including a default
Once in the shell we can try selecting all ``<link>`` objects and see that it
doesn't work (because the Atom XML namespace is obfuscating those nodes):
>>> response.xpath("//link")
[]
.. code-block:: pycon
>>> response.xpath("//link")
[]
But once we call the :meth:`Selector.remove_namespaces` method, all
nodes can be accessed directly by their names:
>>> response.selector.remove_namespaces()
>>> response.xpath("//link")
[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
<Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
...
.. code-block:: pycon
>>> response.selector.remove_namespaces()
>>> response.xpath("//link")
[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
<Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
...
If you wonder why the namespace removal procedure isn't always called by default
instead of having to call it manually, this is because of two reasons, which, in order
@ -735,23 +847,25 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
... <li class="item-1"><a href="link2.html">second item</a></li>
... <li class="item-inactive"><a href="link3.html">third item</a></li>
... <li class="item-1"><a href="link4.html">fourth item</a></li>
... <li class="item-0"><a href="link5.html">fifth item</a></li>
... </ul>
... </div>
... """
>>> sel = Selector(text=doc, type="html")
>>> sel.xpath('//li//@href').getall()
['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
['link1.html', 'link2.html', 'link4.html', 'link5.html']
.. code-block:: pycon
>>> from scrapy import Selector
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
... <li class="item-1"><a href="link2.html">second item</a></li>
... <li class="item-inactive"><a href="link3.html">third item</a></li>
... <li class="item-1"><a href="link4.html">fourth item</a></li>
... <li class="item-0"><a href="link5.html">fifth item</a></li>
... </ul>
... </div>
... """
>>> sel = Selector(text=doc, type="html")
>>> sel.xpath("//li//@href").getall()
['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
['link1.html', 'link2.html', 'link4.html', 'link5.html']
.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
@ -765,7 +879,9 @@ These can be handy for excluding parts of a document tree before
extracting text elements for example.
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
with groups of itemscopes and corresponding itemprops:
.. code-block:: pycon
>>> doc = """
... <div itemscope itemtype="http://schema.org/Product">
@ -776,19 +892,15 @@ with groups of itemscopes and corresponding itemprops::
... Rated <span itemprop="ratingValue">3.5</span>/5
... based on <span itemprop="reviewCount">11</span> customer reviews
... </div>
...
... <div itemprop="offers" itemscope itemtype="http://schema.org/Offer">
... <span itemprop="price">$55.00</span>
... <link itemprop="availability" href="http://schema.org/InStock" />In stock
... </div>
...
... Product description:
... <span itemprop="description">0.7 cubic feet countertop microwave.
... Has six preset cooking categories and convenience features like
... Add-A-Minute and Child Lock.</span>
...
... Customer reviews:
...
... <div itemprop="review" itemscope itemtype="http://schema.org/Review">
... <span itemprop="name">Not a happy camper</span> -
... by <span itemprop="author">Ellie</span>,
@ -801,7 +913,6 @@ with groups of itemscopes and corresponding itemprops::
... <span itemprop="description">The lamp burned out and now I have to replace
... it. </span>
... </div>
...
... <div itemprop="review" itemscope itemtype="http://schema.org/Review">
... <span itemprop="name">Value purchase</span> -
... by <span itemprop="author">Lucas</span>,
@ -818,13 +929,16 @@ with groups of itemscopes and corresponding itemprops::
... </div>
... """
>>> sel = Selector(text=doc, type="html")
>>> for scope in sel.xpath('//div[@itemscope]'):
... print("current scope:", scope.xpath('@itemtype').getall())
... props = scope.xpath('''
>>> for scope in sel.xpath("//div[@itemscope]"):
... print("current scope:", scope.xpath("@itemtype").getall())
... props = scope.xpath(
... """
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
... .//*[@itemscope]/*/@itemprop)"""
... )
... print(f" properties: {props.getall()}")
... print("")
...
current scope: ['http://schema.org/Product']
properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
@ -876,13 +990,15 @@ For the following HTML::
You can use it like this:
>>> response.xpath('//p[has-class("foo")]')
[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
<Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
>>> response.xpath('//p[has-class("foo", "bar-baz")]')
[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
>>> response.xpath('//p[has-class("foo", "bar")]')
[]
.. code-block:: pycon
>>> response.xpath('//p[has-class("foo")]')
[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
<Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
>>> response.xpath('//p[has-class("foo", "bar-baz")]')
[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
>>> response.xpath('//p[has-class("foo", "bar")]')
[]
So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
``p.foo.bar-baz``. Please note, that it is slower in most of the cases,
@ -981,25 +1097,34 @@ Selector examples on HTML response
Here are some :class:`Selector` examples to illustrate several concepts.
In all cases, we assume there is already a :class:`Selector` instantiated with
a :class:`~scrapy.http.HtmlResponse` object like this::
a :class:`~scrapy.http.HtmlResponse` object like this:
.. code-block:: python
sel = Selector(html_response)
1. Select all ``<h1>`` elements from an HTML response body, returning a list of
:class:`Selector` objects (i.e. a :class:`SelectorList` object)::
:class:`Selector` objects (i.e. a :class:`SelectorList` object):
.. code-block:: python
sel.xpath("//h1")
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of strings::
returning a list of strings:
sel.xpath("//h1").getall() # this includes the h1 tag
.. code-block:: python
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag
3. Iterate over all ``<p>`` tags and print their class attribute::
3. Iterate over all ``<p>`` tags and print their class attribute:
.. code-block:: python
for node in sel.xpath("//p"):
print(node.attrib['class'])
print(node.attrib["class"])
.. _selector-examples-xml:
@ -1008,17 +1133,23 @@ Selector examples on XML response
---------------------------------
Here are some examples to illustrate concepts for :class:`Selector` objects
instantiated with an :class:`~scrapy.http.XmlResponse` object::
instantiated with an :class:`~scrapy.http.XmlResponse` object:
.. code-block:: python
sel = Selector(xml_response)
1. Select all ``<product>`` elements from an XML response body, returning a list
of :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
of :class:`Selector` objects (i.e. a :class:`SelectorList` object):
.. code-block:: python
sel.xpath("//product")
2. Extract all prices from a `Google Base XML feed`_ which requires registering
a namespace::
a namespace:
.. code-block:: python
sel.register_namespace("g", "http://base.google.com/ns/1.0")
sel.xpath("//g:price").getall()

202
docs/topics/settings.rst
View File

@ -67,13 +67,15 @@ Example::
Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
own settings that will take precedence and override the project ones. They can
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:
.. code-block:: python
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
custom_settings = {
'SOME_SETTING': 'some value',
"SOME_SETTING": "some value",
}
3. Project settings module
@ -115,14 +117,17 @@ class or a function, there are two different ways you can specify that object:
- As the object itself
For example::
For example:
.. code-block:: python
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
"mybot.pipelines.validate.ValidateMyItem": 300,
}
.. note:: Passing non-callable objects is not supported.
@ -133,11 +138,13 @@ How to access settings
.. highlight:: python
In a spider, the settings are available through ``self.settings``::
In a spider, the settings are available through ``self.settings``:
.. code-block:: python
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://example.com']
name = "myspider"
start_urls = ["http://example.com"]
def parse(self, response):
print(f"Existing settings: {self.settings.attributes.keys()}")
@ -150,7 +157,9 @@ In a spider, the settings are available through ``self.settings``::
Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
extensions, middlewares and item pipelines:
.. code-block:: python
class MyExtension:
def __init__(self, log_is_enabled=False):
@ -160,7 +169,7 @@ extensions, middlewares and item pipelines::
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
return cls(settings.getbool('LOG_ENABLED'))
return cls(settings.getbool("LOG_ENABLED"))
The settings object can be used like a dict (e.g.,
``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting
@ -365,11 +374,13 @@ Scrapy shell <topics-shell>`.
DEFAULT_REQUEST_HEADERS
-----------------------
Default::
Default:
.. code-block:: python
{
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en",
}
The default headers used for Scrapy HTTP Requests. They're populated in the
@ -404,9 +415,11 @@ Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
a :class:`~scrapy.Request` based on its depth.
The priority of a request is adjusted as follows::
The priority of a request is adjusted as follows:
request.priority = request.priority - ( depth * DEPTH_PRIORITY )
.. code-block:: python
request.priority = request.priority - (depth * DEPTH_PRIORITY)
As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
priority (BFO), while negative values increase request priority (DFO). See
@ -595,23 +608,25 @@ orders. For more info see :ref:`topics-downloader-middleware-setting`.
DOWNLOADER_MIDDLEWARES_BASE
---------------------------
Default::
Default:
.. code-block:: python
{
'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
"scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100,
"scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300,
"scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350,
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
"scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
"scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
"scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
"scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
"scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900,
}
A dict containing the downloader middlewares enabled by default in Scrapy. Low
@ -687,15 +702,17 @@ See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
DOWNLOAD_HANDLERS_BASE
----------------------
Default::
Default:
.. code-block:: python
{
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
"data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
"file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
"http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
@ -705,10 +722,12 @@ You should never modify this setting in your project, modify
You can disable any of these download handlers by assigning ``None`` to their
URI scheme in :setting:`DOWNLOAD_HANDLERS`. E.g., to disable the built-in FTP
handler (without replacement), place this in your ``settings.py``::
handler (without replacement), place this in your ``settings.py``:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'ftp': None,
"ftp": None,
}
.. _http2:
@ -718,10 +737,12 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
#. Update :setting:`DOWNLOAD_HANDLERS` as follows:
.. code-block:: python
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
.. warning::
@ -890,18 +911,20 @@ A dict containing the extensions enabled in your project, and their orders.
EXTENSIONS_BASE
---------------
Default::
Default:
.. code-block:: python
{
'scrapy.extensions.corestats.CoreStats': 0,
'scrapy.extensions.telnet.TelnetConsole': 0,
'scrapy.extensions.memusage.MemoryUsage': 0,
'scrapy.extensions.memdebug.MemoryDebugger': 0,
'scrapy.extensions.closespider.CloseSpider': 0,
'scrapy.extensions.feedexport.FeedExporter': 0,
'scrapy.extensions.logstats.LogStats': 0,
'scrapy.extensions.spiderstate.SpiderState': 0,
'scrapy.extensions.throttle.AutoThrottle': 0,
"scrapy.extensions.corestats.CoreStats": 0,
"scrapy.extensions.telnet.TelnetConsole": 0,
"scrapy.extensions.memusage.MemoryUsage": 0,
"scrapy.extensions.memdebug.MemoryDebugger": 0,
"scrapy.extensions.closespider.CloseSpider": 0,
"scrapy.extensions.feedexport.FeedExporter": 0,
"scrapy.extensions.logstats.LogStats": 0,
"scrapy.extensions.spiderstate.SpiderState": 0,
"scrapy.extensions.throttle.AutoThrottle": 0,
}
A dict containing the extensions available by default in Scrapy, and their
@ -988,11 +1011,13 @@ A dict containing the item pipelines to use, and their orders. Order values are
arbitrary, but it is customary to define them in the 0-1000 range. Lower orders
process before higher orders.
Example::
Example:
.. code-block:: python
ITEM_PIPELINES = {
'mybot.pipelines.validate.ValidateMyItem': 300,
'mybot.pipelines.validate.StoreMyItem': 800,
"mybot.pipelines.validate.ValidateMyItem": 300,
"mybot.pipelines.validate.StoreMyItem": 800,
}
.. setting:: ITEM_PIPELINES_BASE
@ -1417,12 +1442,14 @@ testing spiders. For more info see :ref:`topics-contracts`.
SPIDER_CONTRACTS_BASE
---------------------
Default::
Default:
.. code-block:: python
{
'scrapy.contracts.default.UrlContract' : 1,
'scrapy.contracts.default.ReturnsContract': 2,
'scrapy.contracts.default.ScrapesContract': 3,
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.ReturnsContract": 2,
"scrapy.contracts.default.ScrapesContract": 3,
}
A dict containing the Scrapy contracts enabled by default in Scrapy. You should
@ -1431,10 +1458,12 @@ instead. For more info see :ref:`topics-contracts`.
You can disable any of these contracts by assigning ``None`` to their class
path in :setting:`SPIDER_CONTRACTS`. E.g., to disable the built-in
``ScrapesContract``, place this in your ``settings.py``::
``ScrapesContract``, place this in your ``settings.py``:
.. code-block:: python
SPIDER_CONTRACTS = {
'scrapy.contracts.default.ScrapesContract': None,
"scrapy.contracts.default.ScrapesContract": None,
}
.. setting:: SPIDER_LOADER_CLASS
@ -1483,14 +1512,16 @@ orders. For more info see :ref:`topics-spider-middleware-setting`.
SPIDER_MIDDLEWARES_BASE
-----------------------
Default::
Default:
.. code-block:: python
{
'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
}
A dict containing the spider middlewares enabled by default in Scrapy, and
@ -1506,9 +1537,11 @@ Default: ``[]``
A list of modules where Scrapy will look for spiders.
Example::
Example:
SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev']
.. code-block:: python
SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
.. setting:: STATS_CLASS
@ -1597,60 +1630,65 @@ If a reactor is already installed,
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
In order to use the reactor installed by Scrapy::
In order to use the reactor installed by Scrapy:
.. code-block:: python
import scrapy
from twisted.internet import reactor
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
which raises :exc:`Exception`, becomes::
which raises :exc:`Exception`, becomes:
.. code-block:: python
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
name = "quotes"
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
self.timeout = int(kwargs.pop("timeout", "60"))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['https://quotes.toscrape.com/page/1']
urls = ["https://quotes.toscrape.com/page/1"]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
for quote in response.css("div.quote"):
yield {"text": quote.css("span.text::text").get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
self.crawler.engine.close_spider(self, "timeout")
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which

81
docs/topics/shell.rst
View File

@ -191,44 +191,46 @@ all start with the ``[s]`` prefix)::
After that, we can start playing with the objects:
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
.. code-block:: pycon
>>> fetch("https://old.reddit.com/")
>>> response.xpath("//title/text()").get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'
>>> fetch("https://old.reddit.com/")
>>> request = request.replace(method="POST")
>>> response.xpath("//title/text()").get()
'reddit: the front page of the internet'
>>> fetch(request)
>>> request = request.replace(method="POST")
>>> response.status
404
>>> fetch(request)
>>> from pprint import pprint
>>> response.status
404
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
>>> from pprint import pprint
>>> pprint(response.headers)
{'Accept-Ranges': ['bytes'],
'Cache-Control': ['max-age=0, must-revalidate'],
'Content-Type': ['text/html; charset=UTF-8'],
'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
'Server': ['snooserv'],
'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
'Vary': ['accept-encoding'],
'Via': ['1.1 varnish'],
'X-Cache': ['MISS'],
'X-Cache-Hits': ['0'],
'X-Content-Type-Options': ['nosniff'],
'X-Frame-Options': ['SAMEORIGIN'],
'X-Moose': ['majestic'],
'X-Served-By': ['cache-cdg8730-CDG'],
'X-Timer': ['S1481214079.394283,VS0,VE159'],
'X-Ua-Compatible': ['IE=edge'],
'X-Xss-Protection': ['1; mode=block']}
.. _topics-shell-inspect-response:
@ -242,7 +244,9 @@ getting there.
This can be achieved by using the ``scrapy.shell.inspect_response`` function.
Here's an example of how you would call it from your spider::
Here's an example of how you would call it from your spider:
.. code-block:: python
import scrapy
@ -259,6 +263,7 @@ Here's an example of how you would call it from your spider::
# We want to inspect one specific response.
if ".org" in response.url:
from scrapy.shell import inspect_response
inspect_response(response, self)
# Rest of parsing code.
@ -276,14 +281,18 @@ When you run the spider, you will get something similar to this::
Then, you can check if the extraction code is working:
>>> response.xpath('//h1[@class="fn"]')
[]
.. code-block:: pycon
>>> response.xpath('//h1[@class="fn"]')
[]
Nope, it doesn't. So you can open the response in your web browser and see if
it's the response you were expecting:
>>> view(response)
True
.. code-block:: pycon
>>> view(response)
True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::

31
docs/topics/signals.rst
View File

@ -16,7 +16,9 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action::
Here is a simple example showing how you can catch signals and perform some action:
.. code-block:: python
from scrapy import signals
from scrapy import Spider
@ -30,17 +32,14 @@ Here is a simple example showing how you can catch signals and perform some acti
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
return spider
def spider_closed(self, spider):
spider.logger.info('Spider closed: %s', spider.name)
spider.logger.info("Spider closed: %s", spider.name)
def parse(self, response):
pass
@ -56,11 +55,13 @@ you to run asynchronous code that does not block Scrapy. If a signal
handler returns one of these objects, Scrapy waits for that asynchronous
operation to finish.
Let's take an example using :ref:`coroutines <topics-coroutines>`::
Let's take an example using :ref:`coroutines <topics-coroutines>`:
.. code-block:: python
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['https://quotes.toscrape.com/page/1/']
name = "signals"
start_urls = ["https://quotes.toscrape.com/page/1/"]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
@ -71,19 +72,19 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`::
async def item_scraped(self, item):
# Send the scraped item to the server
response = await treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
"http://example.com/post",
json.dumps(item).encode("ascii"),
headers={b"Content-Type": [b"application/json"]},
)
return response
def parse(self, response):
for quote in response.css('div.quote'):
for quote in response.css("div.quote"):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
"text": quote.css("span.text::text").get(),
"author": quote.css("small.author::text").get(),
"tags": quote.css("div.tags a.tag::text").getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support

18
docs/topics/spider-middleware.rst
View File

@ -18,10 +18,12 @@ To activate a spider middleware component, add it to the
:setting:`SPIDER_MIDDLEWARES` setting, which is a dict whose keys are the
middleware class path and their values are the middleware orders.
Here's an example::
Here's an example:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
"myproject.middlewares.CustomSpiderMiddleware": 543,
}
The :setting:`SPIDER_MIDDLEWARES` setting is merged with the
@ -44,11 +46,13 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
value. For example, if you want to disable the off-site middleware:
.. code-block:: python
SPIDER_MIDDLEWARES = {
'myproject.middlewares.CustomSpiderMiddleware': 543,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
"myproject.middlewares.CustomSpiderMiddleware": 543,
"scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None,
}
Finally, keep in mind that some middlewares may need to be enabled through a
@ -261,7 +265,9 @@ specify which response codes the spider is able to handle using the
:setting:`HTTPERROR_ALLOWED_CODES` setting.
For example, if you want your spider to handle 404 responses you can do
this::
this:
.. code-block:: python
class MySpider(CrawlSpider):
handle_httpstatus_list = [404]

246
docs/topics/spiders.rst
View File

@ -157,15 +157,21 @@ scrapy.Spider
If you want to change the Requests used to start scraping a domain, this is
the method to override. For example, if you need to start by logging in using
a POST request, you could do::
a POST request, you could do:
.. code-block:: python
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
return [scrapy.FormRequest("http://www.example.com/login",
formdata={'user': 'john', 'pass': 'secret'},
callback=self.logged_in)]
return [
scrapy.FormRequest(
"http://www.example.com/login",
formdata={"user": "john", "pass": "secret"},
callback=self.logged_in,
)
]
def logged_in(self, response):
# here you would extract links to follow and return Requests for
@ -200,63 +206,71 @@ scrapy.Spider
Called when the spider closes. This method provides a shortcut to
signals.connect() for the :signal:`spider_closed` signal.
Let's see an example::
Let's see an example:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
self.logger.info('A response from %s just arrived!', response.url)
self.logger.info("A response from %s just arrived!", response.url)
Return multiple Requests and items from a single callback::
Return multiple Requests and items from a single callback:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
'http://www.example.com/3.html',
"http://www.example.com/1.html",
"http://www.example.com/2.html",
"http://www.example.com/3.html",
]
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield {"title": h3}
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :class:`~scrapy.Item` objects::
to give data more structure you can use :class:`~scrapy.Item` objects:
.. code-block:: python
import scrapy
from myproject.items import MyItem
class MySpider(scrapy.Spider):
name = 'example.com'
allowed_domains = ['example.com']
name = "example.com"
allowed_domains = ["example.com"]
def start_requests(self):
yield scrapy.Request('http://www.example.com/1.html', self.parse)
yield scrapy.Request('http://www.example.com/2.html', self.parse)
yield scrapy.Request('http://www.example.com/3.html', self.parse)
yield scrapy.Request("http://www.example.com/1.html", self.parse)
yield scrapy.Request("http://www.example.com/2.html", self.parse)
yield scrapy.Request("http://www.example.com/3.html", self.parse)
def parse(self, response):
for h3 in response.xpath('//h3').getall():
for h3 in response.xpath("//h3").getall():
yield MyItem(title=h3)
for href in response.xpath('//a/@href').getall():
for href in response.xpath("//a/@href").getall():
yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@ -274,34 +288,42 @@ Spider arguments are passed through the :command:`crawl` command using the
scrapy crawl myspider -a category=electronics
Spiders can access arguments in their `__init__` methods::
Spiders can access arguments in their `__init__` methods:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def __init__(self, category=None, *args, **kwargs):
super(MySpider, self).__init__(*args, **kwargs)
self.start_urls = [f'http://www.example.com/categories/{category}']
self.start_urls = [f"http://www.example.com/categories/{category}"]
# ...
The default `__init__` method will take any spider arguments
and copy them to the spider as attributes.
The above example can also be written as follows::
The above example can also be written as follows:
.. code-block:: python
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
name = "myspider"
def start_requests(self):
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
yield scrapy.Request(f"http://www.example.com/categories/{self.category}")
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
.. code-block:: python
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
@ -337,10 +359,13 @@ common scraping cases, like following all links on a site based on certain
rules, crawling from `Sitemaps`_, or parsing an XML/CSV feed.
For the examples used in the following spiders, we'll assume you have a project
with a ``TestItem`` declared in a ``myproject.items`` module::
with a ``TestItem`` declared in a ``myproject.items`` module:
.. code-block:: python
import scrapy
class TestItem(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
@ -436,38 +461,46 @@ Crawling rules
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
Let's now take a look at an example CrawlSpider with rules::
Let's now take a look at an example CrawlSpider with rules:
.. code-block:: python
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
class MySpider(CrawlSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com']
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com"]
rules = (
# Extract links matching 'category.php' (but not matching 'subsection.php')
# and follow links from them (since no callback means follow=True by default).
Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
Rule(LinkExtractor(allow=("category\.php",), deny=("subsection\.php",))),
# Extract links matching 'item.php' and parse them with the spider's method parse_item
Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
Rule(LinkExtractor(allow=("item\.php",)), callback="parse_item"),
)
def parse_item(self, response):
self.logger.info('Hi, this is an item page! %s', response.url)
self.logger.info("Hi, this is an item page! %s", response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
item['link_text'] = response.meta['link_text']
item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)")
item["name"] = response.xpath('//td[@id="item_name"]/text()').get()
item["description"] = response.xpath(
'//td[@id="item_description"]/text()'
).get()
item["link_text"] = response.meta["link_text"]
url = response.xpath('//td[@id="additional_data"]/@href').get()
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
return response.follow(
url, self.parse_additional_page, cb_kwargs=dict(item=item)
)
def parse_additional_page(self, response, item):
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
item["additional_data"] = response.xpath(
'//p[@id="additional_data"]/text()'
).get()
return item
@ -568,25 +601,30 @@ XMLFeedSpider
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
These spiders are pretty easy to use, let's have a look at one example::
These spiders are pretty easy to use, let's have a look at one example:
.. code-block:: python
from scrapy.spiders import XMLFeedSpider
from myproject.items import TestItem
class MySpider(XMLFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.xml']
iterator = 'iternodes' # This is actually unnecessary, since it's the default value
itertag = 'item'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.xml"]
iterator = "iternodes" # This is actually unnecessary, since it's the default value
itertag = "item"
def parse_node(self, response, node):
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
self.logger.info(
"Hi, this is a <%s> node!: %s", self.itertag, "".join(node.getall())
)
item = TestItem()
item['id'] = node.xpath('@id').get()
item['name'] = node.xpath('name').get()
item['description'] = node.xpath('description').get()
item["id"] = node.xpath("@id").get()
item["name"] = node.xpath("name").get()
item["description"] = node.xpath("description").get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@ -627,26 +665,29 @@ CSVFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
Let's see an example similar to the previous one, but using a
:class:`CSVFeedSpider`::
:class:`CSVFeedSpider`:
.. code-block:: python
from scrapy.spiders import CSVFeedSpider
from myproject.items import TestItem
class MySpider(CSVFeedSpider):
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.csv']
delimiter = ';'
name = "example.com"
allowed_domains = ["example.com"]
start_urls = ["http://www.example.com/feed.csv"]
delimiter = ";"
quotechar = "'"
headers = ['id', 'name', 'description']
headers = ["id", "name", "description"]
def parse_row(self, response, row):
self.logger.info('Hi, this is a row!: %r', row)
self.logger.info("Hi, this is a row!: %r", row)
item = TestItem()
item['id'] = row['id']
item['name'] = row['name']
item['description'] = row['description']
item["id"] = row["id"]
item["name"] = row["name"]
item["description"] = row["description"]
return item
@ -728,19 +769,22 @@ SitemapSpider
<lastmod>2005-01-01</lastmod>
</url>
We can define a ``sitemap_filter`` function to filter ``entries`` by date::
We can define a ``sitemap_filter`` function to filter ``entries`` by date:
.. code-block:: python
from datetime import datetime
from scrapy.spiders import SitemapSpider
class FilteredSitemapSpider(SitemapSpider):
name = 'filtered_sitemap_spider'
allowed_domains = ['example.com']
sitemap_urls = ['http://example.com/sitemap.xml']
name = "filtered_sitemap_spider"
allowed_domains = ["example.com"]
sitemap_urls = ["http://example.com/sitemap.xml"]
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year >= 2005:
yield entry
@ -765,60 +809,72 @@ SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
Simplest example: process all urls discovered through sitemaps using the
``parse`` callback::
``parse`` callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
def parse(self, response):
pass # ... scrape item here ...
pass # ... scrape item here ...
Process some urls with certain callback and other urls with a different
callback::
callback:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/sitemap.xml']
sitemap_urls = ["http://www.example.com/sitemap.xml"]
sitemap_rules = [
('/product/', 'parse_product'),
('/category/', 'parse_category'),
("/product/", "parse_product"),
("/category/", "parse_category"),
]
def parse_product(self, response):
pass # ... scrape product ...
pass # ... scrape product ...
def parse_category(self, response):
pass # ... scrape category ...
pass # ... scrape category ...
Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps
whose url contains ``/sitemap_shop``::
whose url contains ``/sitemap_shop``:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
sitemap_follow = ['/sitemap_shops']
sitemap_follow = ["/sitemap_shops"]
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
Combine SitemapSpider with other sources of urls::
Combine SitemapSpider with other sources of urls:
.. code-block:: python
from scrapy.spiders import SitemapSpider
class MySpider(SitemapSpider):
sitemap_urls = ['http://www.example.com/robots.txt']
sitemap_urls = ["http://www.example.com/robots.txt"]
sitemap_rules = [
('/shop/', 'parse_shop'),
("/shop/", "parse_shop"),
]
other_urls = ['http://www.example.com/about']
other_urls = ["http://www.example.com/about"]
def start_requests(self):
requests = list(super(MySpider, self).start_requests())
@ -826,10 +882,10 @@ Combine SitemapSpider with other sources of urls::
return requests
def parse_shop(self, response):
pass # ... scrape shop here ...
pass # ... scrape shop here ...
def parse_other(self, response):
pass # ... scrape other here ...
pass # ... scrape other here ...
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index

41
docs/topics/stats.rst
View File

@ -30,10 +30,11 @@ Common Stats Collector uses
===========================
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
attribute. Here is an example of an extension that access stats:
.. code-block:: python
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats
@ -41,31 +42,43 @@ attribute. Here is an example of an extension that access stats::
def from_crawler(cls, crawler):
return cls(crawler.stats)
Set stat value::
Set stat value:
stats.set_value('hostname', socket.gethostname())
.. code-block:: python
Increment stat value::
stats.set_value("hostname", socket.gethostname())
stats.inc_value('custom_count')
Increment stat value:
Set stat value only if greater than previous::
.. code-block:: python
stats.max_value('max_items_scraped', value)
stats.inc_value("custom_count")
Set stat value only if lower than previous::
Set stat value only if greater than previous:
stats.min_value('min_free_memory_percent', value)
.. code-block:: python
stats.max_value("max_items_scraped", value)
Set stat value only if lower than previous:
.. code-block:: python
stats.min_value("min_free_memory_percent", value)
Get stat value:
>>> stats.get_value('custom_count')
1
.. code-block:: pycon
>>> stats.get_value("custom_count")
1
Get all stats:
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
.. code-block:: pycon
>>> stats.get_stats()
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================

2
docs/utils/linkfix.py
View File

@ -18,7 +18,6 @@ from pathlib import Path
def main():
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
@ -50,7 +49,6 @@ def main():
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
Path(_filename).write_text(_contents, encoding="utf-8")

7
extras/qps-bench-server.py
View File

@ -1,9 +1,10 @@
#!/usr/bin/env python
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from time import time
from twisted.internet import reactor
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET, Site
class Root(Resource):

3
extras/qpsclient.py
View File

@ -8,12 +8,11 @@ usage:
"""
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.spiders import Spider
class QPSSpider(Spider):
name = "qps"
benchurl = "http://localhost:8880/"

4
pylintrc
View File

@ -12,6 +12,7 @@ disable=abstract-method,
bad-mcs-classmethod-argument,
bare-except,
broad-except,
broad-exception-raised,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
@ -46,12 +47,14 @@ disable=abstract-method,
method-hidden,
missing-docstring,
no-else-raise,
no-else-return,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-value-for-parameter,
not-callable,
pointless-exception-statement,
pointless-statement,
pointless-string-statement,
protected-access,
@ -87,6 +90,7 @@ disable=abstract-method,
unused-private-member,
unused-variable,
unused-wildcard-import,
use-dict-literal,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,

2
scrapy/VERSION
View File

@ -1 +1 @@
2.7.1
2.8.0

7
scrapy/__init__.py
View File

@ -9,11 +9,10 @@ import warnings
from twisted import version as _txv
# Declare top-level shortcuts
from scrapy.spiders import Spider
from scrapy.http import Request, FormRequest
from scrapy.http import FormRequest, Request
from scrapy.item import Field, Item
from scrapy.selector import Selector
from scrapy.item import Item, Field
from scrapy.spiders import Spider
__all__ = [
"__version__",

9
scrapy/cmdline.py
View File

@ -1,16 +1,17 @@
import sys
import os
import argparse
import cProfile
import inspect
import os
import sys
import pkg_resources
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.project import get_project_settings, inside_project
from scrapy.utils.python import garbage_collect

7
scrapy/commands/__init__.py
View File

@ -1,20 +1,19 @@
"""
Base class for Scrapy commands
"""
import os
import argparse
import os
from pathlib import Path
from typing import Any, Dict, Optional
from twisted.python import failure
from scrapy.crawler import CrawlerProcess
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
requires_project = False
crawler_process: Optional[CrawlerProcess] = None

3
scrapy/commands/bench.py
View File

@ -1,6 +1,6 @@
import subprocess
import sys
import time
import subprocess
from urllib.parse import urlencode
import scrapy
@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractor
class Command(ScrapyCommand):
default_settings = {
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,

5
scrapy/commands/check.py
View File

@ -1,11 +1,12 @@
import time
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
from scrapy.utils.misc import load_object, set_environ
class TextTestResult(_TextTestResult):

1
scrapy/commands/crawl.py
View File

@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError
class Command(BaseRunSpiderCommand):
requires_project = True
def syntax(self):

3
scrapy/commands/edit.py
View File

@ -1,12 +1,11 @@
import sys
import os
import sys
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}

6
scrapy/commands/fetch.py
View File

@ -1,15 +1,15 @@
import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
requires_project = False
def syntax(self):

6
scrapy/commands/genspider.py
View File

@ -1,16 +1,15 @@
import os
import shutil
import string
from pathlib import Path
from importlib import import_module
from pathlib import Path
from typing import Optional, cast
from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
def sanitize_module_name(module_name):
@ -33,7 +32,6 @@ def extract_domain(url):
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False}

1
scrapy/commands/list.py
View File

@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}

6
scrapy/commands/parse.py
View File

@ -2,15 +2,15 @@ import json
import logging
from typing import Dict
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from twisted.internet.defer import maybeDeferred
from w3lib.url import is_url
from twisted.internet.defer import maybeDeferred
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.exceptions import UsageError
logger = logging.getLogger(__name__)

7
scrapy/commands/runspider.py
View File

@ -1,13 +1,13 @@
import sys
from importlib import import_module
from os import PathLike
from pathlib import Path
from importlib import import_module
from types import ModuleType
from typing import Union
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.utils.spider import iter_spider_classes
def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}

1
scrapy/commands/settings.py
View File

@ -5,7 +5,6 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}

3
scrapy/commands/shell.py
View File

@ -8,12 +8,11 @@ from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
class Command(ScrapyCommand):
requires_project = False
default_settings = {
"KEEP_ALIVE": True,

8
scrapy/commands/startproject.py
View File

@ -1,16 +1,15 @@
import re
import os
import re
import string
from importlib.util import find_spec
from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
from scrapy.utils.template import render_templatefile, string_camelcase
TEMPLATES_TO_RENDER = (
("scrapy.cfg",),
@ -29,7 +28,6 @@ def _make_writable(path):
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}

1
scrapy/commands/version.py
View File

@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):

1
scrapy/commands/view.py
View File

@ -1,4 +1,5 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser

2
scrapy/contracts/default.py
View File

@ -1,6 +1,6 @@
import json
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail

13
scrapy/core/downloader/__init__.py
View File

@ -1,16 +1,16 @@
import random
from time import time
from datetime import datetime
from collections import deque
from datetime import datetime
from time import time
from twisted.internet import defer, task
from scrapy import signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.resolver import dnscache
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
class Slot:
@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader:
DOWNLOAD_SLOT = "download_slot"
def __init__(self, crawler):

8
scrapy/core/downloader/contextfactory.py
View File

@ -3,10 +3,10 @@ import warnings
from OpenSSL import SSL
from twisted.internet._sslverify import _setAcceptableProtocols
from twisted.internet.ssl import (
optionsForClientTLS,
CertificateOptions,
platformTrust,
AcceptableCiphers,
CertificateOptions,
optionsForClientTLS,
platformTrust,
)
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
@ -15,8 +15,8 @@ from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
DEFAULT_CIPHERS,
openssl_methods,
ScrapyClientTLSOptions,
openssl_methods,
)
from scrapy.utils.misc import create_instance, load_object

1
scrapy/core/downloader/handlers/__init__.py
View File

@ -10,7 +10,6 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)

7
scrapy/core/downloader/handlers/http11.py
View File

@ -13,15 +13,15 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
URI,
)
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
from zope.interface import implementer
from scrapy import signals
@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent):
class ScrapyAgent:
_Agent = Agent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent

1
scrapy/core/downloader/handlers/http2.py
View File

@ -16,7 +16,6 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
H2DownloadHandlerOrSubclass = TypeVar(
"H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
)

3
scrapy/core/downloader/middleware.py
View File

@ -12,12 +12,11 @@ from scrapy import Spider
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
class DownloaderMiddlewareManager(MiddlewareManager):
component_name = "downloader middleware"
@classmethod

4
scrapy/core/downloader/tls.py
View File

@ -4,12 +4,12 @@ from OpenSSL import SSL
from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import (
ClientTLSOptions,
verifyHostname,
VerificationError,
verifyHostname,
)
from twisted.internet.ssl import AcceptableCiphers
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
from scrapy.utils.ssl import get_temp_key_info, x509name_to_string
logger = logging.getLogger(__name__)

8
scrapy/core/downloader/webclient.py
View File

@ -1,15 +1,15 @@
import re
from time import time
from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.http import HTTPClient
from urllib.parse import urldefrag, urlparse, urlunparse
from twisted.internet import defer
from twisted.internet.protocol import ClientFactory
from twisted.web.http import HTTPClient
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
@ -40,7 +40,6 @@ def _parse(url):
class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b"\n"
def connectionMade(self):
@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient):
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overridden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
waiting = 1

13
scrapy/core/engine.py
View File

@ -10,13 +10,13 @@ from time import time
from typing import (
Any,
Callable,
cast,
Generator,
Iterable,
Iterator,
Optional,
Set,
Union,
cast,
)
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
@ -26,19 +26,14 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import (
CloseSpider,
DontCloseSpider,
ScrapyDeprecationWarning,
)
from scrapy.http import Response, Request
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.reactor import CallLaterOnce
logger = logging.getLogger(__name__)

2
scrapy/core/http2/agent.py
View File

@ -10,7 +10,7 @@ from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpoint
from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider

5
scrapy/core/http2/protocol.py
View File

@ -9,9 +9,9 @@ from h2.config import H2Configuration
from h2.connection import H2Connection
from h2.errors import ErrorCodes
from h2.events import (
Event,
ConnectionTerminated,
DataReceived,
Event,
ResponseReceived,
SettingsAcknowledged,
StreamEnded,
@ -23,7 +23,7 @@ from h2.exceptions import FrameTooLargeError, H2Error
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
from twisted.internet.protocol import connectionDone, Factory, Protocol
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.internet.ssl import Certificate
from twisted.protocols.policies import TimeoutMixin
from twisted.python.failure import Failure
@ -35,7 +35,6 @@ from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
logger = logging.getLogger(__name__)

4
scrapy/core/http2/stream.py
View File

@ -1,13 +1,13 @@
import logging
from enum import Enum
from io import BytesIO
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
from urllib.parse import urlparse
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
from h2.errors import ErrorCodes
from h2.exceptions import H2Error, ProtocolError, StreamClosedError
from hpack import HeaderTuple
from twisted.internet.defer import Deferred, CancelledError
from twisted.internet.defer import CancelledError, Deferred
from twisted.internet.error import ConnectionClosed
from twisted.python.failure import Failure
from twisted.web.client import ResponseFailed

1
scrapy/core/scheduler.py
View File

@ -12,7 +12,6 @@ from scrapy.spiders import Spider
from scrapy.utils.job import job_dir
from scrapy.utils.misc import create_instance, load_object
logger = logging.getLogger(__name__)

6
scrapy/core/scraper.py
View File

@ -5,6 +5,7 @@ from __future__ import annotations
import logging
from collections import deque
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
AsyncIterable,
@ -13,7 +14,6 @@ from typing import (
Iterable,
Optional,
Set,
TYPE_CHECKING,
Tuple,
Union,
)
@ -22,7 +22,7 @@ from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
@ -34,12 +34,10 @@ from scrapy.utils.defer import (
parallel,
parallel_async,
)
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
if TYPE_CHECKING:
from scrapy.crawler import Crawler

6
scrapy/core/spidermw.py
View File

@ -28,14 +28,13 @@ from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
mustbe_deferred,
deferred_from_coro,
deferred_f_from_coro_f,
deferred_from_coro,
maybe_deferred_to_future,
mustbe_deferred,
)
from scrapy.utils.python import MutableAsyncChain, MutableChain
logger = logging.getLogger(__name__)
@ -47,7 +46,6 @@ def _isiterable(o) -> bool:
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
def __init__(self, *middlewares):

6
scrapy/crawler.py
View File

@ -17,20 +17,20 @@ except ImportError:
from zope.interface.verify import verifyClass
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import overridden_settings, Settings
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.utils.log import (
LogCounterHandler,
configure_logging,
get_scrapy_root_handler,
install_scrapy_root_handler,
log_reactor_info,
log_scrapy_info,
LogCounterHandler,
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names

4
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,12 +1,11 @@
import re
import logging
import re
from w3lib import html
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse
logger = logging.getLogger(__name__)
@ -31,7 +30,6 @@ class AjaxCrawlMiddleware:
return cls(crawler.settings)
def process_response(self, request, response, spider):
if not isinstance(response, HtmlResponse) or response.status != 200:
return response

1
scrapy/downloadermiddlewares/decompression.py
View File

@ -14,7 +14,6 @@ from warnings import warn
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.responsetypes import responsetypes
warn(
"scrapy.downloadermiddlewares.decompression is deprecated",
ScrapyDeprecationWarning,

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -23,12 +23,10 @@ from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import load_object
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (
defer.TimeoutError,
TimeoutError,

1
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -53,7 +53,6 @@ class HttpCompressionMiddleware:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
if request.method == "HEAD":
return response
if isinstance(response, Response):

2
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -1,6 +1,6 @@
import base64
from urllib.parse import unquote, urlunparse
from urllib.request import getproxies, proxy_bypass, _parse_proxy
from urllib.request import _parse_proxy, getproxies, proxy_bypass
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached

4
scrapy/downloadermiddlewares/redirect.py
View File

@ -3,10 +3,10 @@ from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import HtmlResponse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs):
class BaseRedirectMiddleware:
enabled_setting = "REDIRECT_ENABLED"
def __init__(self, settings):
@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings):

4
scrapy/downloadermiddlewares/retry.py
View File

@ -9,7 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non failed) pages.
"""
from logging import getLogger, Logger
from logging import Logger, getLogger
from typing import Optional, Union
from twisted.internet import defer
@ -31,7 +31,6 @@ from scrapy.spiders import Spider
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
retry_logger = getLogger(__name__)
@ -123,7 +122,6 @@ def get_retry_request(
class RetryMiddleware:
# IOError is raised by the HttpCompression middleware when trying to
# decompress an empty response
EXCEPTIONS_TO_RETRY = (

7
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -7,8 +7,10 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
import logging
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import load_object
@ -38,6 +40,8 @@ class RobotsTxtMiddleware:
def process_request(self, request, spider):
if request.meta.get("dont_obey_robotstxt"):
return
if request.url.startswith("data:") or request.url.startswith("file:"):
return
d = maybeDeferred(self.robot_parser, request, spider)
d.addCallback(self.process_request_2, request, spider)
return d
@ -69,6 +73,7 @@ class RobotsTxtMiddleware:
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
meta={"dont_obey_robotstxt": True},
callback=NO_CALLBACK,
)
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)

3
scrapy/dupefilters.py
View File

@ -10,8 +10,7 @@ from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, RequestFingerprinter
from scrapy.utils.request import RequestFingerprinter, referer_str
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")

3
scrapy/exporters.py
View File

@ -11,14 +11,13 @@ import warnings
from collections.abc import Mapping
from xml.sax.saxutils import XMLGenerator
from itemadapter import is_item, ItemAdapter
from itemadapter import ItemAdapter, is_item
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
__all__ = [
"BaseItemExporter",
"PprintItemExporter",

1
scrapy/extension.py
View File

@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_list
class ExtensionManager(MiddlewareManager):
component_name = "extension"
@classmethod

6
scrapy/extensions/debug.py
View File

@ -4,11 +4,11 @@ Extensions for debugging Scrapy
See documentation in docs/topics/extensions.rst
"""
import sys
import signal
import logging
import traceback
import signal
import sys
import threading
import traceback
from pdb import Pdb
from scrapy.utils.engine import format_engine_status

5
scrapy/extensions/feedexport.py
View File

@ -16,9 +16,9 @@ from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from zope.interface import implementer, Interface
from zope.interface import Interface, implementer
from scrapy import signals, Spider
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
from scrapy.utils.boto import is_botocore_available
@ -28,7 +28,6 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)

3
scrapy/extensions/httpcache.py
View File

@ -7,7 +7,7 @@ from pathlib import Path
from time import time
from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.http import Headers, Response
from scrapy.http.request import Request
@ -41,7 +41,6 @@ class DummyPolicy:
class RFC2616Policy:
MAXAGE = 3600 * 24 * 365 # one year
def __init__(self, settings):

2
scrapy/extensions/logstats.py
View File

@ -2,8 +2,8 @@ import logging
from twisted.internet import task
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.exceptions import NotConfigured
logger = logging.getLogger(__name__)

6
scrapy/extensions/memusage.py
View File

@ -3,11 +3,11 @@ MemoryUsage extension
See documentation in docs/topics/extensions.rst
"""
import sys
import socket
import logging
from pprint import pformat
import socket
import sys
from importlib import import_module
from pprint import pformat
from twisted.internet import task

Some files were not shown because too many files have changed in this diff Show More