mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into case-insensitive-dict
This commit is contained in:
Eugenio Lacuesta
commit
7297ae566f
|
|
@ -8,6 +8,7 @@ skips:
|
|||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B324
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.5.0
|
||||
current_version = 2.6.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- python-version: 3.9
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: security
|
||||
- python-version: 3.9
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: flake8
|
||||
# Pylint requires installing reppy, which does not support Python 3.9
|
||||
|
|
@ -19,11 +19,10 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
- python-version: 3.9
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: typing
|
||||
- python-version: 3.8 # Keep in sync with .readthedocs.yml
|
||||
- python-version: "3.10" # Keep in sync with .readthedocs.yml
|
||||
env:
|
||||
TOXENV: docs
|
||||
|
||||
|
|
@ -38,8 +37,5 @@ jobs:
|
|||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python 3.9
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Check Tag
|
||||
id: check-release-tag
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [3.6, 3.7, 3.8, 3.9]
|
||||
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -17,10 +17,16 @@ jobs:
|
|||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
- python-version: pypy3
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
PYPY_VERSION: 3.6-v7.3.1
|
||||
PYPY_VERSION: 3.6-v7.3.3
|
||||
|
||||
# pinned deps
|
||||
- python-version: 3.6.12
|
||||
|
|
@ -40,15 +46,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
|
||||
# 3.10-pre
|
||||
- python-version: "3.10.0-beta.4"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10.0-beta.4"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -75,9 +72,6 @@ jobs:
|
|||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
|
|
|
|||
|
|
@ -17,10 +17,15 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: py
|
||||
# https://twistedmatrix.com/trac/ticket/9990
|
||||
#- python-version: 3.9
|
||||
#env:
|
||||
#TOXENV: py
|
||||
- python-version: 3.9
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -5,12 +5,13 @@ sphinx:
|
|||
fail_on_warning: true
|
||||
|
||||
build:
|
||||
image: latest
|
||||
os: ubuntu-20.04
|
||||
tools:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
|
||||
python: "3.10" # Keep in sync with .github/workflows/checks.yml
|
||||
|
||||
python:
|
||||
# For available versions, see:
|
||||
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
|
||||
version: 3.8 # Keep in sync with .github/workflows/checks.yml
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- path: .
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. image:: https://scrapy.org/img/scrapylogo.png
|
||||
|
||||
======
|
||||
Scrapy
|
||||
======
|
||||
|
|
|
|||
|
|
@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest):
|
|||
pytest.skip('This test is only run with --reactor=asyncio')
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def only_not_asyncio(request, reactor_pytest):
|
||||
if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
|
||||
pytest.skip('This test is only run without --reactor=asyncio')
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
if config.getoption("--reactor") == "asyncio":
|
||||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
|
|
|
|||
|
|
@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
|
|||
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
|
||||
|
||||
# Never documented before, and deprecated now.
|
||||
r'^scrapy\.item\.DictItem$',
|
||||
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
|
||||
|
||||
# Implementation detail of LxmlLinkExtractor
|
||||
|
|
@ -304,10 +303,14 @@ intersphinx_mapping = {
|
|||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"command": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
"reqmeta": "tooltip",
|
||||
"setting": "tooltip",
|
||||
"signal": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
|
|||
|
||||
from scrapy.http.response.html import HtmlResponse
|
||||
from sybil import Sybil
|
||||
from sybil.parsers.codeblock import CodeBlockParser
|
||||
try:
|
||||
# >2.0.1
|
||||
from sybil.parsers.codeblock import PythonCodeBlockParser
|
||||
except ImportError:
|
||||
from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
|
||||
from sybil.parsers.doctest import DocTestParser
|
||||
from sybil.parsers.skip import skip
|
||||
|
||||
|
|
@ -21,7 +25,7 @@ def setup(namespace):
|
|||
pytest_collect_file = Sybil(
|
||||
parsers=[
|
||||
DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
|
||||
CodeBlockParser(future_imports=['print_function']),
|
||||
PythonCodeBlockParser(future_imports=['print_function']),
|
||||
skip,
|
||||
],
|
||||
pattern='*.rst',
|
||||
|
|
|
|||
|
|
@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
|
|||
|
||||
See :ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
|
||||
Scrapy crashes with: ImportError: No module named win32api
|
||||
----------------------------------------------------------
|
||||
|
||||
You need to install `pywin32`_ because of `this Twisted bug`_.
|
||||
|
||||
.. _pywin32: https://sourceforge.net/projects/pywin32/
|
||||
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
|
||||
|
||||
How can I simulate a user login in my spider?
|
||||
---------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ testing.
|
|||
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
|
||||
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
|
||||
|
||||
.. _getting-help:
|
||||
|
||||
Getting help
|
||||
============
|
||||
|
||||
|
|
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
|
|||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
|
||||
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
|
||||
|
||||
|
||||
First steps
|
||||
|
|
@ -155,7 +159,6 @@ Solving specific problems
|
|||
topics/debug
|
||||
topics/contracts
|
||||
topics/practices
|
||||
topics/avoiding-bans
|
||||
topics/broad-crawls
|
||||
topics/developer-tools
|
||||
topics/dynamic-content
|
||||
|
|
@ -180,9 +183,6 @@ Solving specific problems
|
|||
:doc:`topics/practices`
|
||||
Get familiar with some Scrapy common practices.
|
||||
|
||||
:doc:`topics/avoiding-bans`
|
||||
Avoid getting banned from websites.
|
||||
|
||||
:doc:`topics/broad-crawls`
|
||||
Tune Scrapy for crawling a lot domains in parallel.
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Examples
|
|||
The best way to learn is with examples, and Scrapy is no exception. For this
|
||||
reason, there is an example Scrapy project named quotesbot_, that you can use to
|
||||
play and learn more about Scrapy. It contains two spiders for
|
||||
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
expressions.
|
||||
|
||||
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
|
|||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
* *(Recommended)* **Don't** use system Python. Install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
|
|
@ -234,8 +234,8 @@ For PyPy3, only Linux installation was tested.
|
|||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependencies will be built during installation.
|
||||
On macOS, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
On macOS, you are likely to face an issue with building the Cryptography
|
||||
dependency. The solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing Scrapy). Installing on Linux has no special
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
Scrapy at a glance
|
||||
==================
|
||||
|
||||
Scrapy is an application framework for crawling web sites and extracting
|
||||
Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
|
||||
structured data which can be used for a wide range of useful applications, like
|
||||
data mining, information processing or historical archival.
|
||||
|
||||
|
|
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
|
|||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
https://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
'https://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Scrapy Tutorial
|
|||
In this tutorial, we'll assume that Scrapy is already installed on your system.
|
||||
If that's not the case, see :ref:`intro-install`.
|
||||
|
||||
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
|
||||
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
|
||||
that lists quotes from famous authors.
|
||||
|
||||
This tutorial will walk you through these tasks:
|
||||
|
|
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
|
|||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
|
@ -143,9 +143,9 @@ similar to this::
|
|||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
|
@ -184,8 +184,8 @@ for your spider::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -207,7 +207,7 @@ Extracting data
|
|||
The best way to learn how to extract data with Scrapy is trying selectors
|
||||
using the :ref:`Scrapy shell <topics-shell>`. Run::
|
||||
|
||||
scrapy shell 'http://quotes.toscrape.com/page/1/'
|
||||
scrapy shell 'https://quotes.toscrape.com/page/1/'
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "http://quotes.toscrape.com/page/1/"
|
||||
scrapy shell "https://quotes.toscrape.com/page/1/"
|
||||
|
||||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 http://quotes.toscrape.com/page/1/>
|
||||
[s] request <GET https://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 https://quotes.toscrape.com/page/1/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
|
||||
[s] Useful shortcuts:
|
||||
|
|
@ -241,7 +241,7 @@ object:
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
|
@ -277,9 +277,19 @@ As an alternative, you could've written:
|
|||
>>> response.css('title::text')[0].get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
|
||||
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
|
||||
find any element matching the selection.
|
||||
Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will
|
||||
raise an :exc:`IndexError` exception if there are no results::
|
||||
|
||||
>>> response.css('noelement')[0].get()
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
IndexError: list index out of range
|
||||
|
||||
You might want to use ``.get()`` directly on the
|
||||
:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None``
|
||||
if there are no results::
|
||||
|
||||
>>> response.css("noelement").get()
|
||||
|
||||
There's a lesson here: for most scraping code, you want it to be resilient to
|
||||
errors due to things not being found on a page, so that even if some parts fail
|
||||
|
|
@ -345,7 +355,7 @@ Extracting quotes and authors
|
|||
Now that you know a bit about selection and extraction, let's complete our
|
||||
spider by writing the code to extract the quotes from the web page.
|
||||
|
||||
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
|
||||
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
|
||||
like this:
|
||||
|
||||
.. code-block:: html
|
||||
|
|
@ -369,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'http://quotes.toscrape.com'
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
@ -434,8 +444,8 @@ in the callback, as you can see below::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -448,9 +458,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
@ -478,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily
|
|||
append new records to it. It doesn't have the same problem of JSON when you run
|
||||
twice. Also, as each record is a separate line, you can process big files
|
||||
without having to fit everything in memory, there are tools like `JQ`_ to help
|
||||
doing that at the command-line.
|
||||
do that at the command-line.
|
||||
|
||||
In small projects (like the one in this tutorial), that should be enough.
|
||||
However, if you want to perform more complex things with the scraped items, you
|
||||
|
|
@ -495,7 +505,7 @@ Following links
|
|||
===============
|
||||
|
||||
Let's say, instead of just scraping the stuff from the first two pages
|
||||
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
|
||||
Now that you know how to extract data from pages, let's see how to follow links
|
||||
from them.
|
||||
|
|
@ -539,7 +549,7 @@ page, extracting data from it::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -644,7 +654,7 @@ this time for scraping author information::
|
|||
class AuthorSpider(scrapy.Spider):
|
||||
name = 'author'
|
||||
|
||||
start_urls = ['http://quotes.toscrape.com/']
|
||||
start_urls = ['https://quotes.toscrape.com/']
|
||||
|
||||
def parse(self, response):
|
||||
author_page_links = response.css('.author + a')
|
||||
|
|
@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument::
|
|||
name = "quotes"
|
||||
|
||||
def start_requests(self):
|
||||
url = 'http://quotes.toscrape.com/'
|
||||
url = 'https://quotes.toscrape.com/'
|
||||
tag = getattr(self, 'tag', None)
|
||||
if tag is not None:
|
||||
url = url + 'tag/' + tag
|
||||
|
|
@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument::
|
|||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
will only visit URLs from the ``humor`` tag, such as
|
||||
``http://quotes.toscrape.com/tag/humor``.
|
||||
``https://quotes.toscrape.com/tag/humor``.
|
||||
|
||||
You can :ref:`learn more about handling spider arguments here <spiderargs>`.
|
||||
|
||||
|
|
|
|||
543
docs/news.rst
543
docs/news.rst
|
|
@ -3,6 +3,450 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.6.1:
|
||||
|
||||
Scrapy 2.6.1 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Fixes a regression introduced in 2.6.0 that would unset the request method when
|
||||
following redirects.
|
||||
|
||||
|
||||
.. _release-2.6.0:
|
||||
|
||||
Scrapy 2.6.0 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Security fixes for cookie handling <2.6-security-fixes>`
|
||||
|
||||
* Python 3.10 support
|
||||
|
||||
* :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental, and works out-of-the-box on Windows regardless of your Python
|
||||
version
|
||||
|
||||
* Feed exports now support :class:`pathlib.Path` output paths and per-feed
|
||||
:ref:`item filtering <item-filter>` and
|
||||
:ref:`post-processing <post-processing>`
|
||||
|
||||
.. _2.6-security-fixes:
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies from a
|
||||
controlled domain into your cookiejar that could be sent to other domains
|
||||
not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
|
||||
advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The h2_ dependency is now optional, only needed to
|
||||
:ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
|
||||
|
||||
.. _h2: https://pypi.org/project/h2/
|
||||
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
|
||||
for a non-POST request, now overrides the URL query string, instead of
|
||||
being appended to it. (:issue:`2919`, :issue:`3579`)
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
|
||||
the return value of that function, and not the ``params`` input parameter,
|
||||
will determine the feed URI parameters, unless that return value is
|
||||
``None``. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- In :class:`scrapy.core.engine.ExecutionEngine`, methods
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
|
||||
and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
now raise :exc:`RuntimeError` if called before
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
|
||||
|
||||
These methods used to assume that
|
||||
:attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
|
||||
been defined by a prior call to
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
|
||||
raising :exc:`AttributeError` instead.
|
||||
|
||||
- If the API of the configured :ref:`scheduler <topics-scheduler>` does not
|
||||
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
|
||||
other exceptions would be raised at run time. (:issue:`3559`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
|
||||
now been removed. (:issue:`5393`)
|
||||
|
||||
- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
|
||||
been removed. (:issue:`4178`, :issue:`4356`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
|
||||
returning ``None`` or modifying the ``params`` input parameter is now
|
||||
deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
|
||||
:meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
|
||||
:func:`scrapy.utils.request.request_from_dict` function.
|
||||
|
||||
- In :mod:`scrapy.squeues`, the following queue classes are deprecated:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
|
||||
instead use:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueue`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
|
||||
|
||||
- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
|
||||
a time when this class could handle multiple :class:`~scrapy.Spider`
|
||||
objects at a time have been deprecated. (:issue:`5090`)
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
|
||||
is deprecated.
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
|
||||
deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
|
||||
|
||||
- The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
|
||||
is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
|
||||
instead.
|
||||
|
||||
- The ``spider`` parameter is deprecated for the following methods:
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.download`
|
||||
|
||||
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
|
||||
first to set the :class:`~scrapy.Spider` object.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- You can now use :ref:`item filtering <item-filter>` to control which items
|
||||
are exported to each output feed. (:issue:`4575`, :issue:`5178`,
|
||||
:issue:`5161`, :issue:`5203`)
|
||||
|
||||
- You can now apply :ref:`post-processing <post-processing>` to feeds, and
|
||||
:ref:`built-in post-processing plugins <builtin-plugins>` are provided for
|
||||
output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
|
||||
|
||||
- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
|
||||
keys. (:issue:`5383`, :issue:`5384`)
|
||||
|
||||
- Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
|
||||
or later will automatically switch the asyncio event loop to one that
|
||||
allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
|
||||
:issue:`5315`)
|
||||
|
||||
- The :command:`genspider` command now supports a start URL instead of a
|
||||
domain name. (:issue:`4439`)
|
||||
|
||||
- :mod:`scrapy.utils.defer` gained 2 new functions,
|
||||
:func:`~scrapy.utils.defer.deferred_to_future` and
|
||||
:func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
|
||||
on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
|
||||
(:issue:`5288`)
|
||||
|
||||
- :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
|
||||
support for `temporary security credentials`_
|
||||
(:setting:`AWS_SESSION_TOKEN`) and endpoint customization
|
||||
(:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
|
||||
(:issue:`5279`)
|
||||
|
||||
- :attr:`Request.cookies <scrapy.Request.cookies>` values that are
|
||||
:class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
|
||||
(:issue:`5252`, :issue:`5253`)
|
||||
|
||||
- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
|
||||
the :signal:`spider_idle` signal to customize the reason why the spider is
|
||||
stopping. (:issue:`5191`)
|
||||
|
||||
- When using
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
|
||||
proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
|
||||
scheme. (:issue:`4505`, :issue:`4649`)
|
||||
|
||||
- All built-in queues now expose a ``peek`` method that returns the next
|
||||
queue object (like ``pop``) but does not remove the returned object from
|
||||
the queue. (:issue:`5112`)
|
||||
|
||||
If the underlying queue does not support peeking (e.g. because you are not
|
||||
using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
|
||||
:exc:`NotImplementedError`.
|
||||
|
||||
- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
|
||||
an ``attributes`` attribute that makes subclassing easier. For
|
||||
:class:`~scrapy.http.Request`, it also allows subclasses to work with
|
||||
:func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
|
||||
:issue:`5130`, :issue:`5218`)
|
||||
|
||||
- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
|
||||
:meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
|
||||
:ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
|
||||
|
||||
- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
|
||||
exceptions now only truncate response bodies longer than 1000 characters,
|
||||
instead of those longer than 32 characters, making it easier to debug such
|
||||
errors. (:issue:`4881`, :issue:`5007`)
|
||||
|
||||
- :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
|
||||
(:issue:`5145`, :issue:`5269`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
|
||||
are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
|
||||
(:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Removed a module-level Twisted reactor import that could prevent
|
||||
:ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
|
||||
|
||||
- The :command:`startproject` command works with existing folders again.
|
||||
(:issue:`4665`, :issue:`4676`)
|
||||
|
||||
- The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
|
||||
(:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
|
||||
``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
|
||||
|
||||
- Made :func:`scrapy.utils.response.open_in_browser` support more complex
|
||||
HTML. (:issue:`5319`, :issue:`5320`)
|
||||
|
||||
- Fixed :attr:`CSVFeedSpider.quotechar
|
||||
<scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
|
||||
encoding. (:issue:`5391`, :issue:`5394`)
|
||||
|
||||
- Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
|
||||
|
||||
.. _setuptools: https://pypi.org/project/setuptools/
|
||||
|
||||
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
now also works as expected with links that have comma-separated ``rel``
|
||||
attribute values including ``nofollow``. (:issue:`5225`)
|
||||
|
||||
- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
|
||||
<topics-feed-exports>` parameter parsing. (:issue:`5359`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental. (:issue:`5332`)
|
||||
|
||||
- Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
|
||||
(:issue:`4976`, :issue:`5315`)
|
||||
|
||||
- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
|
||||
(:issue:`4484`, :issue:`4613`)
|
||||
|
||||
- Documented :ref:`local file naming in media pipelines
|
||||
<topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
|
||||
|
||||
- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
|
||||
:issue:`3669`)
|
||||
|
||||
- Provided better context and instructions to disable the
|
||||
:setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
|
||||
|
||||
- Documented that :ref:`reppy-parser` does not support Python 3.9+.
|
||||
(:issue:`5226`, :issue:`5231`)
|
||||
|
||||
- Documented :ref:`the scheduler component <topics-scheduler>`.
|
||||
(:issue:`3537`, :issue:`3559`)
|
||||
|
||||
- Documented the method used by :ref:`media pipelines
|
||||
<topics-media-pipeline>` to :ref:`determine if a file has expired
|
||||
<file-expiration>`. (:issue:`5120`, :issue:`5254`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now features
|
||||
:func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now covers what happens when you define
|
||||
different per-spider values for some settings that cannot differ at run
|
||||
time. (:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Extended the documentation of the
|
||||
:class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
|
||||
(:issue:`5199`, :issue:`5217`)
|
||||
|
||||
- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
|
||||
:issue:`5224`)
|
||||
|
||||
- Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
|
||||
(:issue:`5174`, :issue:`5244`)
|
||||
|
||||
- Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
|
||||
(:issue:`1582`)
|
||||
|
||||
- Added the ``body_length`` parameter to the documented signature of the
|
||||
:signal:`headers_received` signal. (:issue:`5270`)
|
||||
|
||||
- Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
|
||||
in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
|
||||
|
||||
- The documentation now features the shortest import path of classes with
|
||||
multiple import paths. (:issue:`2733`, :issue:`5099`)
|
||||
|
||||
- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
|
||||
(:issue:`5395`, :issue:`5396`)
|
||||
|
||||
- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
|
||||
to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
|
||||
|
||||
- The pronunciation of the project name is now :ref:`officially
|
||||
<intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
|
||||
|
||||
- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
|
||||
|
||||
- Fixed issues and implemented minor improvements. (:issue:`3155`,
|
||||
:issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
|
||||
:issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
|
||||
:issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
|
||||
:issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
|
||||
|
||||
|
||||
Quality Assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
|
||||
:issue:`5265`)
|
||||
|
||||
- Significantly reduced memory usage by
|
||||
:func:`scrapy.utils.response.response_httprepr`, used by the
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
|
||||
middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
|
||||
|
||||
- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
|
||||
:issue:`5374`)
|
||||
|
||||
- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
|
||||
:issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
|
||||
|
||||
- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
|
||||
:issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
|
||||
:issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
|
||||
:issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
|
||||
:issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
|
||||
:issue:`5425`, :issue:`5427`)
|
||||
|
||||
- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
|
||||
:issue:`5177`, :issue:`5200`)
|
||||
|
||||
- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
|
||||
:issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
|
||||
:issue:`5314`, :issue:`5322`)
|
||||
|
||||
|
||||
.. _release-2.5.1:
|
||||
|
||||
Scrapy 2.5.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-2.5.0:
|
||||
|
||||
Scrapy 2.5.0 (2021-04-06)
|
||||
|
|
@ -940,9 +1384,8 @@ Bug fixes
|
|||
* zope.interface 5.0.0 and later versions are now supported
|
||||
(:issue:`4447`, :issue:`4448`)
|
||||
|
||||
* :meth:`Spider.make_requests_from_url
|
||||
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
|
||||
1.4.0, now issues a warning when used (:issue:`4412`)
|
||||
* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
|
||||
warning when used (:issue:`4412`)
|
||||
|
||||
|
||||
Documentation
|
||||
|
|
@ -1454,6 +1897,88 @@ affect subclasses:
|
|||
(:issue:`3884`)
|
||||
|
||||
|
||||
.. _release-1.8.2:
|
||||
|
||||
Scrapy 1.8.2 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
**Security bug fixes:**
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies into
|
||||
your requests to some other domains. Please, see the `mfjm-vh54-3f96
|
||||
security advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
.. _release-1.8.1:
|
||||
|
||||
Scrapy 1.8.1 (2021-10-05)
|
||||
-------------------------
|
||||
|
||||
* **Security bug fix:**
|
||||
|
||||
If you use
|
||||
:class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
|
||||
(i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
|
||||
authentication, any request exposes your credentials to the request target.
|
||||
|
||||
To prevent unintended exposure of authentication credentials to unintended
|
||||
domains, you must now additionally set a new, additional spider attribute,
|
||||
``http_auth_domain``, and point it to the specific domain to which the
|
||||
authentication credentials must be sent.
|
||||
|
||||
If the ``http_auth_domain`` spider attribute is not set, the domain of the
|
||||
first request will be considered the HTTP authentication target, and
|
||||
authentication credentials will only be sent in requests targeting that
|
||||
domain.
|
||||
|
||||
If you need to send the same HTTP authentication credentials to multiple
|
||||
domains, you can use :func:`w3lib.http.basic_auth_header` instead to
|
||||
set the value of the ``Authorization`` header of your requests.
|
||||
|
||||
If you *really* want your spider to send the same HTTP authentication
|
||||
credentials to any domain, set the ``http_auth_domain`` spider attribute
|
||||
to ``None``.
|
||||
|
||||
Finally, if you are a user of `scrapy-splash`_, know that this version of
|
||||
Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
|
||||
need to upgrade scrapy-splash to a greater version for it to continue to
|
||||
work.
|
||||
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
|
|
@ -1754,7 +2279,7 @@ New features
|
|||
* A new scheduler priority queue,
|
||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targetting multiple web domains, at the
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
|
||||
|
|
@ -2792,7 +3317,7 @@ Bug fixes
|
|||
- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
|
||||
(:issue:`2225`).
|
||||
- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
|
||||
- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
|
||||
|
||||
Refactoring
|
||||
~~~~~~~~~~~
|
||||
|
|
@ -3655,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09)
|
|||
- adding some xpath tips to selectors docs (:commit:`2d103e0`)
|
||||
- fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`)
|
||||
- get_func_args maximum recursion fix #728 (:commit:`81344ea`)
|
||||
- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Updated input/output processor example according to #560. (:commit:`f7c4ea8`)
|
||||
- Fixed Python syntax in tutorial. (:commit:`db59ed9`)
|
||||
- Add test case for tunneling proxy (:commit:`f090260`)
|
||||
- Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`)
|
||||
|
|
@ -4282,7 +4807,7 @@ Code rearranged and removed
|
|||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
|
|
@ -4317,7 +4842,7 @@ Scrapyd changes
|
|||
~~~~~~~~~~~~~~~
|
||||
|
||||
- Scrapyd now uses one process per spider
|
||||
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
|
||||
- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default)
|
||||
- A minimal web ui was added, available at http://localhost:6800 by default
|
||||
- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
|
||||
|
||||
|
|
@ -4353,7 +4878,7 @@ New features and improvements
|
|||
- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
|
||||
- Support for overriding default request headers per spider (#181)
|
||||
- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
|
||||
- Splitted Debian package into two packages - the library and the service (#187)
|
||||
- Split Debian package into two packages - the library and the service (#187)
|
||||
- Scrapy log refactoring (#188)
|
||||
- New extension for keeping persistent spider contexts among different runs (#203)
|
||||
- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ this:
|
|||
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
|
||||
to crawl.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
9. The process repeats (from step 3) until there are no more requests from the
|
||||
:ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
Components
|
||||
|
|
|
|||
|
|
@ -10,10 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
|||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
|
||||
recommended for production environments. Future Scrapy versions
|
||||
may introduce related changes without a deprecation period or
|
||||
warning.
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
|
|
@ -30,6 +26,7 @@ reactor manually. You can do that using
|
|||
|
||||
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
|
||||
|
||||
|
||||
.. _using-custom-loops:
|
||||
|
||||
Using custom asyncio loops
|
||||
|
|
@ -40,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
|
|||
use it instead of the default asyncio event loop.
|
||||
|
||||
|
||||
.. _asyncio-windows:
|
||||
|
||||
Windows-specific notes
|
||||
======================
|
||||
|
||||
The Windows implementation of :mod:`asyncio` can use two event loop
|
||||
implementations:
|
||||
|
||||
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
|
||||
when using Twisted.
|
||||
|
||||
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
|
||||
with Twisted.
|
||||
|
||||
So on Python 3.8+ the event loop class needs to be changed.
|
||||
|
||||
.. versionchanged:: 2.6.0
|
||||
The event loop class is changed automatically when you change the
|
||||
:setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
To change the event loop class manually, call the following code before
|
||||
installing the reactor::
|
||||
|
||||
import asyncio
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
You can put this in the same function that installs the reactor, if you do that
|
||||
yourself, or in some code that runs before the reactor is installed, e.g.
|
||||
``settings.py``.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
subprocesses (this is the case with `playwright`_), so you cannot use
|
||||
them together with Scrapy on Windows (but you should be able to use
|
||||
them on WSL or native Linux).
|
||||
|
||||
.. _playwright: https://github.com/microsoft/playwright-python
|
||||
|
||||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
Awaiting on Deferreds
|
||||
=====================
|
||||
|
||||
When the asyncio reactor isn't installed, you can await on Deferreds in the
|
||||
coroutines directly. When it is installed, this is not possible anymore, due to
|
||||
specifics of the Scrapy coroutine integration (the coroutines are wrapped into
|
||||
:class:`asyncio.Future` objects, not into
|
||||
:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
|
||||
Futures. Scrapy provides two helpers for this:
|
||||
|
||||
.. autofunction:: scrapy.utils.defer.deferred_to_future
|
||||
.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
|
||||
.. tip:: If you need to use these functions in code that aims to be compatible
|
||||
with lower versions of Scrapy that do not provide these functions,
|
||||
down to Scrapy 2.0 (earlier versions do not support
|
||||
:mod:`asyncio`), you can copy the implementation of these functions
|
||||
into your own code.
|
||||
|
|
|
|||
|
|
@ -1,340 +0,0 @@
|
|||
.. _bans:
|
||||
|
||||
=============
|
||||
Avoiding bans
|
||||
=============
|
||||
|
||||
This topic covers some of the strategies that you can follow to avoid getting
|
||||
different or bad responses from a website that you are crawling due to filters
|
||||
such as regional filters, web browser filters, etc.
|
||||
|
||||
.. _avoiding-crawls:
|
||||
|
||||
Avoiding crawls
|
||||
===============
|
||||
|
||||
The best way not to be banned from a website is not to send requests to it in
|
||||
the first place.
|
||||
|
||||
One way to avoid crawling a website is to find the desired dataset through
|
||||
other means. For example, you can use Google’s `dataset search engine`_.
|
||||
|
||||
If the target website is the only or best source of the desired information,
|
||||
and you only need to extract the data on a monthly basis or a lower frequency,
|
||||
you may be able to crawl a public snapshot of the target website instead.
|
||||
`Common Crawl`_ is an open repository of web crawl data that you can access
|
||||
freely. It contains monthly snapshots of a wide variety of websites and, if you
|
||||
are lucky, your target website will be among them.
|
||||
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _dataset search engine: https://datasetsearch.research.google.com/
|
||||
|
||||
|
||||
.. _being-polite:
|
||||
|
||||
Being polite
|
||||
============
|
||||
|
||||
To avoid being banned, you should first avoid giving a website reasons to ban
|
||||
you.
|
||||
|
||||
.. _identifying-yourself:
|
||||
|
||||
Identifying yourself
|
||||
--------------------
|
||||
|
||||
If your crawling has a noticeable negative impact on a website or you crawl
|
||||
content that should not be crawled, website administrators will need to do
|
||||
something.
|
||||
|
||||
Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and
|
||||
includes contact information, so that website administrators can contact you.
|
||||
|
||||
|
||||
.. _following-robotstxt:
|
||||
|
||||
Following robots.txt guidelines
|
||||
-------------------------------
|
||||
|
||||
Some websites provide a ``robots.txt`` file at their root path (e.g.
|
||||
``http://example.com/robots.txt``) that describes the guidelines that they wish
|
||||
bots to follow when crawling their website.
|
||||
|
||||
Before you start writing a spider for a website, read their ``robots.txt``
|
||||
file and implement your spider following its guidelines. See the `robots.txt
|
||||
standard draft`_ or the `robots.txt Google specification`_ for information on
|
||||
how to read ``robots.txt`` files.
|
||||
|
||||
To ensure that your spider does not crawl pages restricted by ``robots.txt``
|
||||
guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
|
||||
middleware. When you do, if your spider attempts to crawl a restricted page,
|
||||
this middleware aborts that request with the following message::
|
||||
|
||||
Forbidden by robots.txt
|
||||
|
||||
Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or
|
||||
``Request-Rate`` directives from the ``robots.txt`` guidelines.
|
||||
|
||||
You may also use the :ref:`AutoThrottle extension <topics-autothrottle>` on top
|
||||
of that, so that when the target website experiences a high load, your spider
|
||||
automatically switches to higher download delays.
|
||||
|
||||
.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt
|
||||
.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00
|
||||
|
||||
|
||||
.. _choosing-crawl-speed:
|
||||
|
||||
Finding the right guidelines on your own
|
||||
----------------------------------------
|
||||
|
||||
If a website does not specify a desired download delay, or does not provide a
|
||||
``robots.txt`` file, you should make an effort to find out the right values for
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that
|
||||
will not have a noticeable negative impact on the target website.
|
||||
|
||||
Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that
|
||||
the target website receives, and choose concurrency and delay values that will
|
||||
not cause a noticeable traffic increase.
|
||||
|
||||
.. _SimilarWeb: https://www.similarweb.com
|
||||
|
||||
|
||||
.. _filters-and-challenges:
|
||||
|
||||
Bypassing filters and solving challenges
|
||||
========================================
|
||||
|
||||
Some websites implement filters and challenges that aim to deny access or alter
|
||||
their content based on aspects of the visitor, such as the country where they
|
||||
are or the web browsing tool they use.
|
||||
|
||||
.. _regional-filter:
|
||||
|
||||
Bypassing regional filters
|
||||
--------------------------
|
||||
|
||||
Some websites send different or bad responses based on the region or country
|
||||
associated to your `IP address`_.
|
||||
|
||||
To bypass these filters, get access to a `proxy server`_ that has an outgoing
|
||||
IP address from a region that gets the desired responses.
|
||||
|
||||
Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
|
||||
middleware to configure your spider to use that proxy.
|
||||
|
||||
.. _IP address: https://en.wikipedia.org/wiki/IP_address
|
||||
.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server
|
||||
|
||||
|
||||
.. _web-browser-filter:
|
||||
|
||||
Bypassing web browser filters
|
||||
-----------------------------
|
||||
|
||||
Some websites send different or bad responses if they detect that your request
|
||||
does not come from a web browser.
|
||||
|
||||
To bypass these filters, switch your :setting:`USER_AGENT` to a value copied
|
||||
from those that popular web browsers use. In some rare cases, you may need a
|
||||
user agent string from a specific web browser.
|
||||
|
||||
There are multiple Scrapy plugins that can rotate your requests through popular
|
||||
web browser user agent strings, such as scrapy-fake-useragent_,
|
||||
scrapy-random-useragent_ or Scrapy-UserAgents_.
|
||||
|
||||
For advanced web browser filters,
|
||||
:ref:`pre-rendering JavaScript <topics-javascript-rendering>` or
|
||||
:ref:`using a headless browser <topics-headless-browsing>` may be necessary.
|
||||
Use these options only as a last resort, however, because they cause a higher
|
||||
load per request on the target website.
|
||||
|
||||
.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent
|
||||
.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent
|
||||
.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/
|
||||
|
||||
|
||||
.. _request-delay-filter:
|
||||
|
||||
Bypassing request delay filters
|
||||
-------------------------------
|
||||
|
||||
Some websites may ban your IP after they detect that your requests use a
|
||||
constant download delay.
|
||||
|
||||
To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
|
||||
setting is enabled by default. When that is not enough, an
|
||||
:ref:`IP address rotation solution <ip-rotation>` may be much more effective.
|
||||
|
||||
|
||||
.. _isp-filter:
|
||||
|
||||
Bypassing internet service provider filters
|
||||
-------------------------------------------
|
||||
|
||||
Some websites send different or bad responses if they detect that your request
|
||||
comes from an IP address that belongs to a `data center`_, as opposed to a
|
||||
residential IP address from an `internet service provider`_ or a mobile IP
|
||||
address from a `mobile network`_.
|
||||
|
||||
To bypass these filters, get access to a proxy server that has an outgoing IP
|
||||
address that is either residential or mobile. Note that you may also get
|
||||
different responses depending on whether your IP address is residential or
|
||||
mobile.
|
||||
|
||||
Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
|
||||
middleware to configure your spider to use that proxy.
|
||||
|
||||
.. _data center: https://en.wikipedia.org/wiki/Data_center
|
||||
.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider
|
||||
.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network
|
||||
|
||||
|
||||
.. _captcha:
|
||||
|
||||
Solving CAPTCHA challenges
|
||||
--------------------------
|
||||
|
||||
Some websites require you to solve a `CAPTCHA challenge`_ to get the desired
|
||||
response.
|
||||
|
||||
To bypass these filters, several options exist:
|
||||
|
||||
- You could have your spider present the CAPTCHA challenge to you and wait
|
||||
for you to solve it manually.
|
||||
|
||||
- Some CAPTCHA challenges can be solved using an `optical character
|
||||
recognition`_ (OCR) solution such as pytesseract_.
|
||||
|
||||
- Paid CAPTCHA solving services exist.
|
||||
|
||||
Whichever solution you choose, implement it as a :ref:`downloader middleware
|
||||
<topics-downloader-middleware>` that automatically detects CAPTCHA challenges
|
||||
in responses and solves them, so that your spider code only receives successful
|
||||
responses.
|
||||
|
||||
.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA
|
||||
.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition
|
||||
.. _pytesseract: https://github.com/madmaze/pytesseract
|
||||
|
||||
|
||||
.. _ip-rotation:
|
||||
|
||||
IP address rotation solutions
|
||||
=============================
|
||||
|
||||
See below some of the different solutions there are to have your requests use
|
||||
different outgoing IP addresses.
|
||||
|
||||
When using this approach, remember to set :setting:`COOKIES_ENABLED` to
|
||||
``False`` to disable global cookie handling. This prevents websites from
|
||||
identifying two requests as coming from the same user agent even if they come
|
||||
from different IP addresses and have different user-agent strings. You can
|
||||
still include some cookies manually in your requests. Define them through the
|
||||
``Cookies`` header of your requests. See
|
||||
:class:`Request.headers <scrapy.http.Request.headers>`.
|
||||
|
||||
.. _smart-proxy:
|
||||
|
||||
Smart proxies
|
||||
-------------
|
||||
|
||||
An increasing number of websites use solutions that apply many of the above
|
||||
filters and challenges at the same time.
|
||||
|
||||
There are paid proxy services, like `Zyte Smart Proxy Manager`_, that
|
||||
automatically bypass website filters and challenges, so that your spider only
|
||||
gets successful responses. They also allow managing sessions to simulate user
|
||||
behavior.
|
||||
|
||||
For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced
|
||||
integration with Scrapy. For other services, use the
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware
|
||||
or implement your own :ref:`downloader middleware
|
||||
<topics-downloader-middleware>`.
|
||||
|
||||
.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
||||
|
||||
.. _rotating-proxy:
|
||||
|
||||
Rotating proxies
|
||||
----------------
|
||||
|
||||
Rotating proxy services like ProxyMesh_ send different requests through
|
||||
different proxies. This can decrease the likelihood of being affected by some
|
||||
filters or challenges.
|
||||
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
|
||||
|
||||
.. _free-proxies:
|
||||
|
||||
Free proxies
|
||||
------------
|
||||
|
||||
You can easily find lists of free proxies in the internet, and you can use
|
||||
a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in
|
||||
your spider and have requests rotate through them automatically.
|
||||
|
||||
This approach, however, has serious drawbacks:
|
||||
|
||||
- Free proxies may stop working at any moment. You need to implement a way to
|
||||
refresh your list of free proxies.
|
||||
|
||||
- In addition to handling occasional bad responses from websites, you
|
||||
need to handle all kinds of bad responses from proxies. You may even need
|
||||
to inspect the response body to determine if a response comes from the
|
||||
target website or from a misbehaving proxy.
|
||||
|
||||
- Advanced antibot solutions may automatically detect and filter out traffic
|
||||
from free proxies.
|
||||
|
||||
.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies
|
||||
|
||||
|
||||
.. _custom-rotating-proxy:
|
||||
|
||||
Custom rotating proxy
|
||||
---------------------
|
||||
|
||||
If you have spare servers, you can set them up as proxies and use scrapoxy_ to
|
||||
build a custom proxy that rotates traffic through them. However, the initial
|
||||
setup can be complex, and your requests will be vulnerable to
|
||||
:ref:`internet service provider filtering <isp-filter>`.
|
||||
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
|
||||
|
||||
.. _tor:
|
||||
|
||||
The Tor network
|
||||
---------------
|
||||
|
||||
It is possible to send requests through the `Tor network`_.
|
||||
|
||||
The initial setup to have Scrapy working with Tor is not straightforward.
|
||||
Use a search engine to find up-to-date documentation specific to using
|
||||
Scrapy and Tor together.
|
||||
|
||||
The main drawback of using the Tor network is that traffic can be extremely
|
||||
slow.
|
||||
|
||||
.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network)
|
||||
|
||||
|
||||
.. _commercial-support:
|
||||
|
||||
Seeking professional help
|
||||
=========================
|
||||
|
||||
Avoiding bans, filters and challenges can be difficult and tricky, and may
|
||||
sometimes require special infrastructure.
|
||||
|
||||
If you find yourself unable to prevent your spider from getting bad responses,
|
||||
consider contacting `commercial support`_.
|
||||
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
|
|
@ -230,10 +230,16 @@ Usage example::
|
|||
genspider
|
||||
---------
|
||||
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
|
||||
* Requires project: *no*
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
.. versionadded:: 2.6.0
|
||||
The ability to pass a URL instead of a domain.
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
|
||||
.. note:: Even if an HTTPS URL is specified, the protocol used in
|
||||
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
|
|
|||
|
|
@ -75,23 +75,28 @@ coroutines, functions that return Deferreds and functions that return
|
|||
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
|
||||
This means you can use many useful Python libraries providing such code::
|
||||
|
||||
class MySpider(Spider):
|
||||
class MySpiderDeferred(Spider):
|
||||
# ...
|
||||
async def parse_with_deferred(self, response):
|
||||
async def parse(self, response):
|
||||
additional_response = await treq.get('https://additional.url')
|
||||
additional_data = await treq.content(additional_response)
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
async def parse_with_asyncio(self, response):
|
||||
class MySpiderAsyncio(Spider):
|
||||
# ...
|
||||
async def parse(self, response):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get('https://additional.url') as additional_response:
|
||||
additional_data = await r.text()
|
||||
additional_data = await additional_response.text()
|
||||
# ... use response and additional_data to yield items and requests
|
||||
|
||||
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
|
||||
:mod:`asyncio` loop and to use them you need to
|
||||
:doc:`enable asyncio support in Scrapy<asyncio>`.
|
||||
|
||||
.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
|
||||
you need to :ref:`wrap them<asyncio-await-dfd>`.
|
||||
|
||||
Common use cases for asynchronous code include:
|
||||
|
||||
* requesting data from websites, databases and other services (in callbacks,
|
||||
|
|
|
|||
|
|
@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
|
|||
|
||||
Since Developer Tools operate on a live browser DOM, what you'll actually see
|
||||
when inspecting the page source is not the original HTML, but a modified one
|
||||
after applying some browser clean up and executing Javascript code. Firefox,
|
||||
after applying some browser clean up and executing JavaScript code. Firefox,
|
||||
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
|
||||
the other hand, does not modify the original page HTML, so you won't be able to
|
||||
extract any data if you use ``<tbody>`` in your XPath expressions.
|
||||
|
||||
Therefore, you should keep in mind the following things:
|
||||
|
||||
* Disable Javascript while inspecting the DOM looking for XPaths to be
|
||||
* Disable JavaScript while inspecting the DOM looking for XPaths to be
|
||||
used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
|
||||
|
||||
* Never use full XPath paths, use relative and clever ones based on attributes
|
||||
|
|
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
|
|||
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
|
||||
copy XPaths to selected elements. Let's try it out.
|
||||
|
||||
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
|
||||
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
$ scrapy shell "http://quotes.toscrape.com/"
|
||||
$ scrapy shell "https://quotes.toscrape.com/"
|
||||
|
||||
Then, back to your web browser, right-click on the ``span`` tag, select
|
||||
``Copy > XPath`` and paste it in the Scrapy shell like so:
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
|
||||
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
|
||||
|
||||
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
|
||||
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
|
||||
|
|
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
|
|||
type ``json``.
|
||||
|
||||
If we click on this request, we see that the request URL is
|
||||
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
is a JSON-object that contains our quotes. We can also right-click
|
||||
on the request and open ``Open in new tab`` to get a better overview.
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
|
|||
name = 'quote'
|
||||
allowed_domains = ['quotes.toscrape.com']
|
||||
page = 1
|
||||
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
|
||||
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
|
||||
|
||||
def parse(self, response):
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
|
|||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
self.page += 1
|
||||
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
This spider starts at the first page of the quotes-API. With each
|
||||
|
|
@ -280,7 +280,7 @@ request::
|
|||
from scrapy import Request
|
||||
|
||||
request = Request.from_curl(
|
||||
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
|
||||
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
|
||||
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
|
||||
|
|
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
|
|||
to identifying the correct request and replicating it in your spider.
|
||||
|
||||
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
|
||||
.. _quotes.toscrape.com: http://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _quotes.toscrape.com: https://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
methods of installed middleware is always called on every response.
|
||||
|
||||
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
|
||||
process_request methods and reschedule the returned request. Once the newly returned
|
||||
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
|
||||
request is performed, the appropriate middleware chain will be called on
|
||||
the downloaded response.
|
||||
|
||||
|
|
@ -323,8 +323,21 @@ HttpAuthMiddleware
|
|||
This middleware authenticates all requests generated from certain spiders
|
||||
using `Basic access authentication`_ (aka. HTTP auth).
|
||||
|
||||
To enable HTTP authentication from certain spiders, set the ``http_user``
|
||||
and ``http_pass`` attributes of those spiders.
|
||||
To enable HTTP authentication for a spider, set the ``http_user`` and
|
||||
``http_pass`` spider attributes to the authentication data and the
|
||||
``http_auth_domain`` spider attribute to the domain which requires this
|
||||
authentication (its subdomains will be also handled in the same way).
|
||||
You can set ``http_auth_domain`` to ``None`` to enable the
|
||||
authentication for all requests but you risk leaking your authentication
|
||||
credentials to unrelated domains.
|
||||
|
||||
.. warning::
|
||||
In previous Scrapy versions HttpAuthMiddleware sent the authentication
|
||||
data with all requests, which is a security problem if the spider
|
||||
makes requests to several different domains. Currently if the
|
||||
``http_auth_domain`` attribute is not set, the middleware will use the
|
||||
domain of the first request, which will work for some spiders but not
|
||||
for others. In the future the middleware will produce an error instead.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -334,6 +347,7 @@ HttpAuthMiddleware
|
|||
|
||||
http_user = 'someuser'
|
||||
http_pass = 'somepass'
|
||||
http_auth_domain = 'intranet.example.com'
|
||||
name = 'intranet.example.com'
|
||||
|
||||
# .. rest of the spider code omitted ...
|
||||
|
|
@ -352,7 +366,7 @@ HttpCacheMiddleware
|
|||
This middleware provides low-level cache to all HTTP requests and responses.
|
||||
It has to be combined with a cache storage backend as well as a cache policy.
|
||||
|
||||
Scrapy ships with three HTTP cache storage backends:
|
||||
Scrapy ships with the following HTTP cache storage backends:
|
||||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
|
|
@ -690,14 +704,15 @@ HttpCompressionMiddleware
|
|||
sent/received from web sites.
|
||||
|
||||
This middleware also supports decoding `brotli-compressed`_ as well as
|
||||
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
|
||||
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
|
||||
installed, respectively.
|
||||
|
||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||
.. _brotlipy: https://pypi.org/project/brotlipy/
|
||||
.. _brotli: https://pypi.org/project/Brotli/
|
||||
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
||||
.. _zstandard: https://pypi.org/project/zstandard/
|
||||
|
||||
|
||||
HttpCompressionMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -1005,7 +1020,7 @@ Parsers vary in several aspects:
|
|||
(shorter) rule
|
||||
|
||||
Performance comparison of different parsers is available at `the following link
|
||||
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
|
||||
<https://github.com/scrapy/scrapy/issues/3969>`_.
|
||||
|
||||
.. _protego-parser:
|
||||
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ Example::
|
|||
class ProductXmlExporter(XmlItemExporter):
|
||||
|
||||
def serialize_field(self, field, name, value):
|
||||
if field == 'price':
|
||||
if name == 'price':
|
||||
return f'$ {str(value)}'
|
||||
return super().serialize_field(field, name, value)
|
||||
|
||||
|
|
|
|||
|
|
@ -272,12 +272,13 @@ in multiple files, with the specified maximum item count per file. That way, as
|
|||
soon as a file reaches the maximum item count, that file is delivered to the
|
||||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
|
||||
.. _item-filter:
|
||||
|
||||
Item filtering
|
||||
==============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
You can filter items that you want to allow for a particular feed by using the
|
||||
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
|
||||
|
|
@ -312,6 +313,63 @@ ItemFilter
|
|||
:members:
|
||||
|
||||
|
||||
.. _post-processing:
|
||||
|
||||
Post-Processing
|
||||
===============
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
the feed to be processed. These plugins can be declared either as an import string
|
||||
or with the imported class of the plugin. Parameters to plugins can be passed
|
||||
through the feed options. See :ref:`feed options <feed-options>` for examples.
|
||||
|
||||
.. _builtin-plugins:
|
||||
|
||||
Built-in Plugins
|
||||
----------------
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
|
||||
|
||||
.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
|
||||
|
||||
.. _custom-plugins:
|
||||
|
||||
Custom Plugins
|
||||
--------------
|
||||
|
||||
Each plugin is a class that must implement the following methods:
|
||||
|
||||
.. method:: __init__(self, file, feed_options)
|
||||
|
||||
Initialize the plugin.
|
||||
|
||||
:param file: file-like object having at least the `write`, `tell` and `close` methods implemented
|
||||
|
||||
:param feed_options: feed-specific :ref:`options <feed-options>`
|
||||
:type feed_options: :class:`dict`
|
||||
|
||||
.. method:: write(self, data)
|
||||
|
||||
Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
|
||||
It must return number of bytes written.
|
||||
|
||||
.. method:: close(self)
|
||||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
|
|
@ -368,10 +426,12 @@ For instance::
|
|||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path('items.csv'): {
|
||||
pathlib.Path('items.csv.gz'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||
'gzip_compresslevel': 5,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -397,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
If undefined or empty, all items are exported.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
|
||||
|
||||
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
|
|
@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
|
||||
|
||||
- ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
|
||||
|
||||
The plugins will be used in the order of the list passed.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -679,6 +744,9 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
|
|
|
|||
|
|
@ -190,6 +190,8 @@ item.
|
|||
|
||||
import scrapy
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
|
||||
class ScreenshotPipeline:
|
||||
"""Pipeline that uses Splash to render screenshot of
|
||||
|
|
@ -202,7 +204,7 @@ item.
|
|||
encoded_item_url = quote(adapter["url"])
|
||||
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
|
||||
request = scrapy.Request(screenshot_url)
|
||||
response = await spider.crawler.engine.download(request, spider)
|
||||
response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
|
||||
|
||||
if response.status != 200:
|
||||
# Error happened, return item.
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
|
|||
|
||||
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
|
||||
|
||||
.. _topics-keeping-persistent-state-between-batches:
|
||||
|
||||
Keeping persistent state between batches
|
||||
========================================
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ chapter <topics-items>`::
|
|||
l.add_xpath('name', '//div[@class="product_name"]')
|
||||
l.add_xpath('name', '//div[@class="product_title"]')
|
||||
l.add_xpath('price', '//p[@id="price"]')
|
||||
l.add_css('stock', 'p#stock]')
|
||||
l.add_css('stock', 'p#stock')
|
||||
l.add_value('last_updated', 'today') # you can also use literal values
|
||||
return l.load_item()
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ Logging settings
|
|||
These settings can be used to configure the logging:
|
||||
|
||||
* :setting:`LOG_FILE`
|
||||
* :setting:`LOG_FILE_APPEND`
|
||||
* :setting:`LOG_ENABLED`
|
||||
* :setting:`LOG_ENCODING`
|
||||
* :setting:`LOG_LEVEL`
|
||||
|
|
@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
|
|||
:setting:`LOG_FILE` is set, messages sent through the root logger will be
|
||||
redirected to a file named :setting:`LOG_FILE` with encoding
|
||||
:setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
|
||||
messages will be displayed on the standard error. Lastly, if
|
||||
messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
|
||||
and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
|
||||
(discarding the output from previous runs, if any). Lastly, if
|
||||
:setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
|
||||
|
||||
:setting:`LOG_LEVEL` determines the minimum level of severity to display, those
|
||||
|
|
@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many
|
|||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
|
|
|
|||
|
|
@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
|||
Additional features
|
||||
===================
|
||||
|
||||
.. _file-expiration:
|
||||
|
||||
File expiration
|
||||
---------------
|
||||
|
||||
|
|
@ -383,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
|
|||
|
||||
and pipeline class MyPipeline will have expiration time set to 180.
|
||||
|
||||
The last modified time from the file is used to determine the age of the file in days,
|
||||
which is then compared to the set expiration time to determine if the file is expired.
|
||||
|
||||
.. _topics-images-thumbnails:
|
||||
|
||||
Thumbnail generation for images
|
||||
|
|
|
|||
|
|
@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
# Your second spider definition
|
||||
...
|
||||
|
||||
configure_logging()
|
||||
settings = get_project_settings()
|
||||
configure_logging(settings)
|
||||
runner = CrawlerRunner(settings)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
crawl()
|
||||
reactor.run() # the script will block here until the last crawl call is finished
|
||||
|
||||
Different spiders can set different values for the same setting, but when they
|
||||
run in the same process it may be impossible, by design or because of some
|
||||
limitations, to use these different values. What happens in practice is
|
||||
different for different settings:
|
||||
|
||||
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
|
||||
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
|
||||
default one) cannot be read from the per-spider settings. These are applied
|
||||
when the :class:`~scrapy.crawler.CrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerProcess` object is created.
|
||||
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
|
||||
available value is used, and if a spider requests a different reactor an
|
||||
exception will be raised. These are applied when the reactor is installed.
|
||||
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
|
||||
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
|
||||
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
|
||||
the first available value is used. These are applied when the reactor is
|
||||
started.
|
||||
|
||||
.. seealso:: :ref:`run-from-script`.
|
||||
|
||||
.. _distributed-crawls:
|
||||
|
|
@ -226,5 +245,39 @@ crawl::
|
|||
curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
|
||||
curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
|
||||
|
||||
.. _bans:
|
||||
|
||||
Avoiding getting banned
|
||||
=======================
|
||||
|
||||
Some websites implement certain measures to prevent bots from crawling them,
|
||||
with varying degrees of sophistication. Getting around those measures can be
|
||||
difficult and tricky, and may sometimes require special infrastructure. Please
|
||||
consider contacting `commercial support`_ if in doubt.
|
||||
|
||||
Here are some tips to keep in mind when dealing with these kinds of sites:
|
||||
|
||||
* rotate your user agent from a pool of well-known ones from browsers (google
|
||||
around to get a list of them)
|
||||
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
|
||||
cookies to spot bot behaviour
|
||||
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
|
||||
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
||||
directly
|
||||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
super proxy that you can attach your own proxies to.
|
||||
* use a highly distributed downloader that circumvents bans internally, so you
|
||||
can just focus on parsing clean pages. One example of such downloaders is
|
||||
`Zyte Smart Proxy Manager`_
|
||||
|
||||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
||||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
|
|
|||
|
|
@ -110,6 +110,10 @@ Request objects
|
|||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
|
||||
are casted to :class:`str`.
|
||||
|
||||
:type cookies: dict or list
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
@ -300,7 +304,7 @@ errors if needed::
|
|||
"http://www.httpbin.org/status/404", # Not found error
|
||||
"http://www.httpbin.org/status/500", # server issue
|
||||
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
|
||||
"http://www.httphttpbinbin.org/", # DNS error expected
|
||||
"https://example.invalid/", # DNS error expected
|
||||
]
|
||||
|
||||
def start_requests(self):
|
||||
|
|
@ -950,6 +954,14 @@ TextResponse objects
|
|||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
.. method:: TextResponse.urljoin(url)
|
||||
|
||||
Constructs an absolute url by combining the Response's base url with
|
||||
a possible relative url. The base url shall be extracted from the
|
||||
``<base>`` tag, or just the Response's :attr:`url` if there is no such
|
||||
tag.
|
||||
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
--------------------
|
||||
|
|
|
|||
|
|
@ -1026,6 +1026,16 @@ Default: ``None``
|
|||
|
||||
File name to use for logging output. If ``None``, standard error will be used.
|
||||
|
||||
.. setting:: LOG_FILE_APPEND
|
||||
|
||||
LOG_FILE_APPEND
|
||||
---------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
If ``False``, the log file specified with :setting:`LOG_FILE` will be
|
||||
overwritten (discarding the output from previous runs, if any).
|
||||
|
||||
.. setting:: LOG_FORMAT
|
||||
|
||||
LOG_FORMAT
|
||||
|
|
@ -1566,7 +1576,7 @@ If a reactor is already installed,
|
|||
|
||||
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
|
||||
:exc:`Exception` if the installed reactor does not match the
|
||||
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
|
||||
:setting:`TWISTED_REACTOR` setting; therefore, having top-level
|
||||
:mod:`~twisted.internet.reactor` imports in project files and imported
|
||||
third-party libraries will make Scrapy raise :exc:`Exception` when
|
||||
it checks which reactor is installed.
|
||||
|
|
@ -1587,7 +1597,7 @@ In order to use the reactor installed by Scrapy::
|
|||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1615,7 +1625,7 @@ which raises :exc:`Exception`, becomes::
|
|||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1628,10 +1638,9 @@ which raises :exc:`Exception`, becomes::
|
|||
|
||||
|
||||
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
|
||||
means that Scrapy will not attempt to install any specific reactor, and the
|
||||
default reactor defined by Twisted for the current platform will be used. This
|
||||
is to maintain backward compatibility and avoid possible problems caused by
|
||||
using a non-default reactor.
|
||||
means that Scrapy will install the default reactor defined by Twisted for the
|
||||
current platform. This is to maintain backward compatibility and avoid possible
|
||||
problems caused by using a non-default reactor.
|
||||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
|
|
@ -1645,8 +1654,19 @@ Default: ``2083``
|
|||
|
||||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs. For more information about
|
||||
the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
The maximum URL length to allow for crawled URLs.
|
||||
|
||||
This setting can act as a stopping condition in case of URLs of ever-increasing
|
||||
length, which may be caused for example by a programming error either in the
|
||||
target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
|
||||
:setting:`DEPTH_LIMIT`.
|
||||
|
||||
Use ``0`` to allow URLs of any length.
|
||||
|
||||
The default value is copied from the `Microsoft Internet Explorer maximum URL
|
||||
length`_, even though this setting exists for different reasons.
|
||||
|
||||
.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
|
||||
|
||||
.. setting:: USER_AGENT
|
||||
|
||||
|
|
@ -1658,7 +1678,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
|
|||
The default User-Agent to use when crawling, unless overridden. This user agent is
|
||||
also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
|
||||
if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
|
||||
there is no overridding User-Agent header specified for the request.
|
||||
there is no overriding User-Agent header specified for the request.
|
||||
|
||||
|
||||
Settings documented elsewhere:
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ Available Shortcuts
|
|||
shortcuts
|
||||
|
||||
- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
|
||||
and update all related objects accordingly. You can optionaly ask for HTTP
|
||||
and update all related objects accordingly. You can optionally ask for HTTP
|
||||
3xx redirections to not be followed by passing ``redirect=False``
|
||||
|
||||
- ``fetch(request)`` - fetch a new response from the given request and update
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ Let's take an example::
|
|||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
start_urls = ['https://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
|
|
@ -413,7 +413,7 @@ headers_received
|
|||
.. versionadded:: 2.5
|
||||
|
||||
.. signal:: headers_received
|
||||
.. function:: headers_received(headers, request, spider)
|
||||
.. function:: headers_received(headers, body_length, request, spider)
|
||||
|
||||
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||
available for a given request, before downloading any additional content.
|
||||
|
|
|
|||
|
|
@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
method (from a previous spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
|
||||
objects.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
executing any other :meth:`process_spider_exception` in the following
|
||||
|
|
@ -440,4 +440,3 @@ UrlLengthMiddleware
|
|||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
|
|
|||
|
|
@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are
|
|||
different kinds of default spiders bundled into Scrapy for different purposes.
|
||||
We will talk about those types here.
|
||||
|
||||
.. module:: scrapy.spiders
|
||||
:synopsis: Spiders base class, spider manager and spider middleware
|
||||
|
||||
.. _topics-spiders-ref:
|
||||
|
||||
scrapy.Spider
|
||||
=============
|
||||
|
||||
.. class:: scrapy.spiders.Spider()
|
||||
.. class:: scrapy.spiders.Spider
|
||||
.. class:: scrapy.Spider()
|
||||
|
||||
This is the simplest spider, and the one from which every other spider
|
||||
|
|
@ -122,6 +119,11 @@ scrapy.Spider
|
|||
send log messages through it as described on
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. attribute:: state
|
||||
|
||||
A dict you can use to persist some spider state between batches.
|
||||
See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
|
||||
|
||||
.. method:: from_crawler(crawler, *args, **kwargs)
|
||||
|
||||
This is the class method used by Scrapy to create your spiders.
|
||||
|
|
@ -367,7 +369,7 @@ CrawlSpider
|
|||
described below. If multiple rules match the same link, the first one
|
||||
will be used, according to the order they're defined in this attribute.
|
||||
|
||||
This spider also exposes an overrideable method:
|
||||
This spider also exposes an overridable method:
|
||||
|
||||
.. method:: parse_start_url(response, **kwargs)
|
||||
|
||||
|
|
@ -529,7 +531,7 @@ XMLFeedSpider
|
|||
itertag = 'n:url'
|
||||
# ...
|
||||
|
||||
Apart from these new attributes, this spider has the following overrideable
|
||||
Apart from these new attributes, this spider has the following overridable
|
||||
methods too:
|
||||
|
||||
.. method:: adapt_response(response)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
|
|||
large changes.
|
||||
* *B* is the release number. This will include many changes including features
|
||||
and things that possibly break backward compatibility, although we strive to
|
||||
keep theses cases at a minimum.
|
||||
keep these cases at a minimum.
|
||||
* *C* is the bugfix release number.
|
||||
|
||||
Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
A spider that generate light requests to meassure QPS troughput
|
||||
A spider that generate light requests to meassure QPS throughput
|
||||
|
||||
usage:
|
||||
|
||||
|
|
|
|||
2
pylintrc
2
pylintrc
|
|
@ -105,12 +105,14 @@ disable=abstract-method,
|
|||
unnecessary-lambda,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unspecified-encoding,
|
||||
unsubscriptable-object,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-private-member,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
use-implicit-booleaness-not-comparison,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
|
|
|
|||
|
|
@ -20,5 +20,4 @@ addopts =
|
|||
--ignore=docs/utils
|
||||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
filterwarnings=
|
||||
ignore::DeprecationWarning:twisted.web.test.test_webclient
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.5.0
|
||||
2.6.1
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
|||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 6):
|
||||
print("Scrapy %s requires Python 3.6+" % __version__)
|
||||
print(f"Scrapy {__version__} requires Python 3.6+")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import argparse
|
||||
import cProfile
|
||||
import inspect
|
||||
import pkg_resources
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
|
|
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
|
||||
parser.description = cmd.long_desc()
|
||||
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler='resolve',
|
||||
description=cmd.long_desc())
|
||||
settings.setdict(cmd.default_settings, priority='command')
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
cmd.crawler_process = CrawlerProcess(settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from optparse import OptionGroup
|
||||
import argparse
|
||||
from typing import Any, Dict
|
||||
|
||||
from twisted.python import failure
|
||||
|
|
@ -43,14 +43,14 @@ class ScrapyCommand:
|
|||
|
||||
def long_desc(self):
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since no post-formatting will
|
||||
"help" command. It can contain newlines since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
|
@ -59,22 +59,20 @@ class ScrapyCommand:
|
|||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
group = parser.add_argument_group(title='Global Options')
|
||||
group.add_argument("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_argument("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_argument("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_argument("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
try:
|
||||
|
|
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_argument("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_argument("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
|
||||
class ScrapyHelpFormatter(argparse.HelpFormatter):
|
||||
"""
|
||||
Help Formatter for scrapy command line help messages.
|
||||
"""
|
||||
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
|
||||
super().__init__(prog, indent_increment=indent_increment,
|
||||
max_help_position=max_help_position, width=width)
|
||||
|
||||
def _join_parts(self, part_strings):
|
||||
parts = self.format_part_strings(part_strings)
|
||||
return super()._join_parts(parts)
|
||||
|
||||
def format_part_strings(self, part_strings):
|
||||
"""
|
||||
Underline and title case command line help message headers.
|
||||
"""
|
||||
if part_strings and part_strings[0].startswith("usage: "):
|
||||
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
|
||||
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
|
||||
for index in headings[::-1]:
|
||||
char = '-' if "Global Options" in part_strings[index] else '='
|
||||
part_strings[index] = part_strings[index][:-2].title()
|
||||
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
|
||||
part_strings.insert(index + 1, underline)
|
||||
return part_strings
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
parser.add_argument("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import string
|
|||
|
||||
from importlib import import_module
|
||||
from os.path import join, dirname, abspath, exists, splitext
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
|
|||
return module_name
|
||||
|
||||
|
||||
def extract_domain(url):
|
||||
"""Extract domain name from URL string"""
|
||||
o = urlparse(url)
|
||||
if o.scheme == '' and o.netloc == '':
|
||||
o = urlparse("//" + url.lstrip("/"))
|
||||
return o.netloc
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
@ -35,16 +44,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_argument("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_argument("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
@ -59,7 +68,8 @@ class Command(ScrapyCommand):
|
|||
if len(args) != 2:
|
||||
raise UsageError()
|
||||
|
||||
name, domain = args[0:2]
|
||||
name, url = args[0:2]
|
||||
domain = extract_domain(url)
|
||||
module = sanitize_module_name(name)
|
||||
|
||||
if self.settings.get('BOT_NAME') == module:
|
||||
|
|
|
|||
|
|
@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
parser.add_argument("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_argument("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_argument("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_argument("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_argument("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
|
|||
|
|
@ -18,16 +18,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
parser.add_argument("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_argument("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_argument("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _start_crawler_thread(self):
|
||||
t = Thread(target=self.crawler_process.start,
|
||||
kwargs={'stop_after_crawl': False})
|
||||
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import re
|
||||
import os
|
||||
import string
|
||||
from importlib import import_module
|
||||
from importlib.util import find_spec
|
||||
from os.path import join, exists, abspath
|
||||
from shutil import ignore_patterns, move, copy2, copystat
|
||||
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
|
||||
|
|
@ -42,11 +42,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _is_valid_name(self, project_name):
|
||||
def _module_exists(module_name):
|
||||
try:
|
||||
import_module(module_name)
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
spec = find_spec(module_name)
|
||||
return spec is not None and spec.loader is not None
|
||||
|
||||
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
|
||||
print('Error: Project names must begin with a letter and contain'
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ class Command(fetch.Command):
|
|||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
parser.add_argument('--headers', help=argparse.SUPPRESS)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
|
|||
settings=settings,
|
||||
crawler=crawler,
|
||||
)
|
||||
msg = """
|
||||
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
|
||||
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
|
||||
Please upgrade your context factory class to handle them or ignore them.""" % (
|
||||
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
|
||||
msg = (
|
||||
f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
|
||||
"a `method` argument (type OpenSSL.SSL method, e.g. "
|
||||
"OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
|
||||
"argument and/or a `tls_ciphers` argument. Please, upgrade your "
|
||||
"context factory class to handle them or ignore them."
|
||||
)
|
||||
warnings.warn(msg)
|
||||
|
||||
return context_factory
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ class TunnelingAgent(Agent):
|
|||
# proxy host and port are required for HTTP pool `key`
|
||||
# otherwise, same remote host connection request could reuse
|
||||
# a cached tunneled connection to a different proxy
|
||||
key = key + self._proxyConf
|
||||
key += self._proxyConf
|
||||
return super()._requestWithEndpoint(
|
||||
key=key,
|
||||
endpoint=endpoint,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
from typing import Callable, Union
|
||||
from typing import Callable, Union, cast
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
@defer.inlineCallbacks
|
||||
def process_request(request: Request):
|
||||
for method in self.methods['process_request']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
def process_exception(failure: Failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
|
|||
|
|
@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
verifyHostname(connection, self._hostnameASCII)
|
||||
except (CertificateError, VerificationError) as e:
|
||||
logger.warning(
|
||||
'Remote certificate is not valid for hostname "{}"; {}'.format(
|
||||
self._hostnameASCII, e))
|
||||
'Remote certificate is not valid for hostname "%s"; %s',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
'Ignoring error while verifying certificate '
|
||||
'from host "{}" (exception: {})'.format(
|
||||
self._hostnameASCII, repr(e)))
|
||||
'from host "%s" (exception: %r)',
|
||||
self._hostnameASCII, e)
|
||||
|
||||
|
||||
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from time import time
|
|||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
|
||||
from scrapy.http import Headers
|
||||
|
|
@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
from twisted.internet import reactor
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
|
|
|||
|
|
@ -285,8 +285,8 @@ class Stream:
|
|||
|
||||
self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
|
||||
|
||||
bytes_to_send_size = bytes_to_send_size - chunk_size
|
||||
self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
|
||||
bytes_to_send_size -= chunk_size
|
||||
self.metadata['remaining_content_length'] -= chunk_size
|
||||
|
||||
self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
|
||||
|
||||
|
|
|
|||
|
|
@ -156,7 +156,7 @@ class Scraper:
|
|||
callback = result.request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
dfd = defer_succeed(result)
|
||||
dfd.addCallback(callback, **result.request.cb_kwargs)
|
||||
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
|
||||
else: # result is a Failure
|
||||
result.request = request
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Spider Middleware manager
|
|||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
from itertools import islice
|
||||
from typing import Any, Callable, Generator, Iterable, Union
|
||||
from typing import Any, Callable, Generator, Iterable, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
|
||||
spider: Spider) -> Any:
|
||||
for method in self.methods['process_spider_input']:
|
||||
method = cast(Callable, method)
|
||||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.utils.log import (
|
|||
configure_logging,
|
||||
get_scrapy_root_handler,
|
||||
install_scrapy_root_handler,
|
||||
log_reactor_info,
|
||||
log_scrapy_info,
|
||||
LogCounterHandler,
|
||||
)
|
||||
|
|
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class Crawler:
|
||||
|
||||
def __init__(self, spidercls, settings=None):
|
||||
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
|
||||
if isinstance(spidercls, Spider):
|
||||
raise ValueError('The spidercls argument must be a class, not an object')
|
||||
|
||||
|
|
@ -69,6 +70,20 @@ class Crawler:
|
|||
|
||||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
# but before something imports twisted.internet.reactor
|
||||
if reactor_class:
|
||||
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
else:
|
||||
from twisted.internet import default
|
||||
default.install()
|
||||
log_reactor_info()
|
||||
if reactor_class:
|
||||
verify_installed_reactor(reactor_class)
|
||||
|
||||
self.extensions = ExtensionManager.from_crawler(self)
|
||||
|
||||
self.settings.freeze()
|
||||
|
|
@ -153,7 +168,6 @@ class CrawlerRunner:
|
|||
self._crawlers = set()
|
||||
self._active = set()
|
||||
self.bootstrap_failed = False
|
||||
self._handle_twisted_reactor()
|
||||
|
||||
@property
|
||||
def spiders(self):
|
||||
|
|
@ -247,10 +261,6 @@ class CrawlerRunner:
|
|||
while self._active:
|
||||
yield defer.DeferredList(self._active)
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
|
||||
|
||||
|
||||
class CrawlerProcess(CrawlerRunner):
|
||||
"""
|
||||
|
|
@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
||||
|
|
@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner):
|
|||
{'signame': signame})
|
||||
reactor.callFromThread(self._stop_reactor)
|
||||
|
||||
def start(self, stop_after_crawl=True):
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings, init_reactor=True)
|
||||
|
||||
def start(self, stop_after_crawl=True, install_signal_handlers=True):
|
||||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
|
|
@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
||||
:param bool install_signal_handlers: whether to install the shutdown
|
||||
handlers (default: True)
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
if stop_after_crawl:
|
||||
|
|
@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner):
|
|||
return
|
||||
d.addBoth(self._stop_reactor)
|
||||
|
||||
if install_signal_handlers:
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
resolver_class = load_object(self.settings["DNS_RESOLVER"])
|
||||
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
|
||||
resolver.install_on_reactor()
|
||||
|
|
@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner):
|
|||
reactor.stop()
|
||||
except RuntimeError: # raised if already stopped or in shutdown stage
|
||||
pass
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from tldextract import TLDExtract
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_split_domain = TLDExtract(include_psl_private_domains=True)
|
||||
|
||||
|
||||
def _is_public_domain(domain):
|
||||
parts = _split_domain(domain)
|
||||
return not parts.domain
|
||||
|
||||
|
||||
class CookiesMiddleware:
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
|
||||
|
|
@ -23,14 +34,29 @@ class CookiesMiddleware:
|
|||
raise NotConfigured
|
||||
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
|
||||
|
||||
def _process_cookies(self, cookies, *, jar, request):
|
||||
for cookie in cookies:
|
||||
cookie_domain = cookie.domain
|
||||
if cookie_domain.startswith('.'):
|
||||
cookie_domain = cookie_domain[1:]
|
||||
|
||||
request_domain = urlparse_cached(request).hostname.lower()
|
||||
|
||||
if cookie_domain and _is_public_domain(cookie_domain):
|
||||
if cookie_domain != request_domain:
|
||||
continue
|
||||
cookie.domain = request_domain
|
||||
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if request.meta.get('dont_merge_cookies', False):
|
||||
return
|
||||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
for cookie in self._get_request_cookies(jar, request):
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
# set Cookie header
|
||||
request.headers.pop('Cookie', None)
|
||||
|
|
@ -44,7 +70,9 @@ class CookiesMiddleware:
|
|||
# extract cookies from Set-Cookie and drop invalid/expired cookies
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
jar.extract_cookies(response, request)
|
||||
cookies = jar.make_cookies(response, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
self._debug_set_cookie(response, spider)
|
||||
|
||||
return response
|
||||
|
|
@ -80,8 +108,8 @@ class CookiesMiddleware:
|
|||
logger.warning(msg.format(request, cookie, key))
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], str):
|
||||
decoded[key] = cookie[key]
|
||||
if isinstance(cookie[key], (bool, float, int, str)):
|
||||
decoded[key] = str(cookie[key])
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.url import url_is_from_any_domain
|
||||
|
||||
|
||||
class HttpAuthMiddleware:
|
||||
|
|
@ -24,8 +28,23 @@ class HttpAuthMiddleware:
|
|||
pwd = getattr(spider, 'http_pass', '')
|
||||
if usr or pwd:
|
||||
self.auth = basic_auth_header(usr, pwd)
|
||||
if not hasattr(spider, 'http_auth_domain'):
|
||||
warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
|
||||
'problems if the spider makes requests to several different domains. http_auth_domain '
|
||||
'will be set to the domain of the first request, please set it to the correct value '
|
||||
'explicitly.',
|
||||
category=ScrapyDeprecationWarning)
|
||||
self.domain_unset = True
|
||||
else:
|
||||
self.domain = spider.http_auth_domain
|
||||
self.domain_unset = False
|
||||
|
||||
def process_request(self, request, spider):
|
||||
auth = getattr(self, 'auth', None)
|
||||
if auth and b'Authorization' not in request.headers:
|
||||
request.headers[b'Authorization'] = auth
|
||||
domain = urlparse_cached(request).hostname
|
||||
if self.domain_unset:
|
||||
self.domain = domain
|
||||
self.domain_unset = False
|
||||
if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
|
||||
request.headers[b'Authorization'] = auth
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class HttpCacheMiddleware:
|
|||
self.stats.inc_value('httpcache/miss', spider=spider)
|
||||
if self.ignore_missing:
|
||||
self.stats.inc_value('httpcache/ignore', spider=spider)
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
raise IgnoreRequest(f"Ignored request not in cache: {request}")
|
||||
return None # first time request
|
||||
|
||||
# Return cached response only if not expired
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse
|
|||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
|
||||
|
|
@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_redirect_request(source_request, *, url, **kwargs):
|
||||
redirect_request = source_request.replace(
|
||||
url=url,
|
||||
**kwargs,
|
||||
cookies=None,
|
||||
)
|
||||
if 'Cookie' in redirect_request.headers:
|
||||
source_request_netloc = urlparse_cached(source_request).netloc
|
||||
redirect_request_netloc = urlparse_cached(redirect_request).netloc
|
||||
if source_request_netloc != redirect_request_netloc:
|
||||
del redirect_request.headers['Cookie']
|
||||
return redirect_request
|
||||
|
||||
|
||||
class BaseRedirectMiddleware:
|
||||
|
||||
enabled_setting = 'REDIRECT_ENABLED'
|
||||
|
|
@ -47,10 +62,15 @@ class BaseRedirectMiddleware:
|
|||
raise IgnoreRequest("max redirections reached")
|
||||
|
||||
def _redirect_request_using_get(self, request, redirect_url):
|
||||
redirected = request.replace(url=redirect_url, method='GET', body='')
|
||||
redirected.headers.pop('Content-Type', None)
|
||||
redirected.headers.pop('Content-Length', None)
|
||||
return redirected
|
||||
redirect_request = _build_redirect_request(
|
||||
request,
|
||||
url=redirect_url,
|
||||
method='GET',
|
||||
body='',
|
||||
)
|
||||
redirect_request.headers.pop('Content-Type', None)
|
||||
redirect_request.headers.pop('Content-Length', None)
|
||||
return redirect_request
|
||||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
|
|
@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
if response.status in (301, 307, 308) or request.method == 'HEAD':
|
||||
redirected = request.replace(url=redirected_url)
|
||||
redirected = _build_redirect_request(request, url=redirected_url)
|
||||
return self._redirect(redirected, request, spider, response.status)
|
||||
|
||||
redirected = self._redirect_request_using_get(request, redirected_url)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
An extension to retry failed requests that are potentially caused by temporary
|
||||
problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
You can change the behaviour of this middleware by modifing the scraping settings:
|
||||
You can change the behaviour of this middleware by modifying the scraping settings:
|
||||
RETRY_TIMES - how many times to retry a failed page
|
||||
RETRY_HTTP_CODES - which HTTP response codes to retry
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,22 @@
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.python import global_object_name, to_bytes
|
||||
from scrapy.utils.request import request_httprepr
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
from twisted.web import http
|
||||
|
||||
|
||||
def get_header_size(headers):
|
||||
size = 0
|
||||
for key, value in headers.items():
|
||||
if isinstance(value, (list, tuple)):
|
||||
for v in value:
|
||||
size += len(b": ") + len(key) + len(v)
|
||||
return size + len(b'\r\n') * (len(headers.keys()) - 1)
|
||||
|
||||
|
||||
def get_status_size(response_status):
|
||||
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
|
||||
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
|
||||
|
||||
|
||||
class DownloaderStats:
|
||||
|
|
@ -24,7 +39,8 @@ class DownloaderStats:
|
|||
def process_response(self, request, response, spider):
|
||||
self.stats.inc_value('downloader/response_count', spider=spider)
|
||||
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
|
||||
reslen = len(response_httprepr(response))
|
||||
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
|
||||
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
|
||||
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ class BaseItemExporter:
|
|||
self._configure(kwargs, dont_fail=dont_fail)
|
||||
|
||||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
"""Configure the exporter by popping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||
"""
|
||||
|
|
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
return serializer(value)
|
||||
|
||||
def _serialize_value(self, value):
|
||||
if isinstance(value, _BaseItem):
|
||||
if isinstance(value, Item):
|
||||
return self.export_item(value)
|
||||
elif is_item(value):
|
||||
return dict(self._serialize_item(value))
|
||||
|
|
|
|||
|
|
@ -11,15 +11,16 @@ import sys
|
|||
import warnings
|
||||
from datetime import datetime
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import implementer, Interface
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
|
|
@ -37,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
|||
kwargs['feed_options'] = feed_options
|
||||
else:
|
||||
warnings.warn(
|
||||
"{} does not support the 'feed_options' keyword argument. Add a "
|
||||
f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove this "
|
||||
"warning. This parameter will become mandatory in a future "
|
||||
"version of Scrapy."
|
||||
.format(builder.__qualname__),
|
||||
"version of Scrapy.",
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return builder(*preargs, uri, *args, **kwargs)
|
||||
|
|
@ -355,32 +355,28 @@ class FeedExporter:
|
|||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
|
||||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
|
||||
)
|
||||
d.addErrback(
|
||||
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
self._handle_store_error, logmsg, spider, type(slot.storage).__name__
|
||||
)
|
||||
return d
|
||||
|
||||
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
|
||||
def _handle_store_error(self, f, logmsg, spider, slot_type):
|
||||
logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
"Error storing %s", logmsg,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
|
||||
|
||||
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
|
||||
def _handle_store_success(self, f, logmsg, spider, slot_type):
|
||||
logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
"Stored %s", logmsg,
|
||||
extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
|
||||
|
||||
|
|
@ -396,6 +392,9 @@ class FeedExporter:
|
|||
"""
|
||||
storage = self._get_storage(uri, feed_options)
|
||||
file = storage.open(spider)
|
||||
if "postprocessing" in feed_options:
|
||||
file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
|
||||
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed_options['format'],
|
||||
|
|
@ -470,10 +469,10 @@ class FeedExporter:
|
|||
for uri_template, values in self.feeds.items():
|
||||
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
|
||||
logger.error(
|
||||
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
|
||||
''.format(uri_template)
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
|
||||
uri_template
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
|
@ -522,10 +521,15 @@ class FeedExporter:
|
|||
instance = build_instance(feedcls)
|
||||
method_name = '__new__'
|
||||
if instance is None:
|
||||
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
|
||||
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
def _get_uri_params(
|
||||
self,
|
||||
spider: Spider,
|
||||
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
|
||||
slot: Optional[_FeedSlot] = None,
|
||||
) -> dict:
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
|
|
@ -533,9 +537,18 @@ class FeedExporter:
|
|||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
original_params = params.copy()
|
||||
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
|
||||
new_params = uripar_function(params, spider)
|
||||
if new_params is None or original_params != params:
|
||||
warnings.warn(
|
||||
'Modifying the params dictionary in-place in the function defined in '
|
||||
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
|
||||
'setting is deprecated. The function must return a new dictionary '
|
||||
'instead.',
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return new_params if new_params is not None else params
|
||||
|
||||
def _load_filter(self, feed_options):
|
||||
# load the item filter if declared else load the default filter class
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ class DbmCacheStorage:
|
|||
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
|
||||
self.db = self.dbmodule.open(dbpath, 'c')
|
||||
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.db.close()
|
||||
|
|
@ -280,7 +280,7 @@ class FilesystemCacheStorage:
|
|||
self._open = gzip.open if self.use_gzip else open
|
||||
|
||||
def open_spider(self, spider):
|
||||
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
|
||||
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
|
||||
extra={'spider': spider})
|
||||
|
||||
def close_spider(self, spider):
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ class MemoryUsage:
|
|||
self.crawler = crawler
|
||||
self.warned = False
|
||||
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
|
||||
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
|
||||
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
|
||||
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
|
||||
self.mail = MailSender.from_settings(crawler.settings)
|
||||
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
|
||||
|
|
@ -77,7 +77,7 @@ class MemoryUsage:
|
|||
def _check_limit(self):
|
||||
if self.get_virtual_size() > self.limit:
|
||||
self.crawler.stats.set_value('memusage/limit_reached', 1)
|
||||
mem = self.limit/1024/1024
|
||||
mem = self.limit / 1024 / 1024
|
||||
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
@ -94,11 +94,11 @@ class MemoryUsage:
|
|||
self.crawler.stop()
|
||||
|
||||
def _check_warning(self):
|
||||
if self.warned: # warn only once
|
||||
if self.warned: # warn only once
|
||||
return
|
||||
if self.get_virtual_size() > self.warning:
|
||||
self.crawler.stats.set_value('memusage/warning_reached', 1)
|
||||
mem = self.warning/1024/1024
|
||||
mem = self.warning / 1024 / 1024
|
||||
logger.warning("Memory usage reached %(memusage)dM",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,154 @@
|
|||
"""
|
||||
Extension for processing data before they are exported to feeds.
|
||||
"""
|
||||
from bz2 import BZ2File
|
||||
from gzip import GzipFile
|
||||
from io import IOBase
|
||||
from lzma import LZMAFile
|
||||
from typing import Any, BinaryIO, Dict, List
|
||||
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class GzipPlugin:
|
||||
"""
|
||||
Compresses received data using `gzip <https://en.wikipedia.org/wiki/Gzip>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `gzip_compresslevel`
|
||||
- `gzip_mtime`
|
||||
- `gzip_filename`
|
||||
|
||||
See :py:class:`gzip.GzipFile` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
compress_level = self.feed_options.get("gzip_compresslevel", 9)
|
||||
mtime = self.feed_options.get("gzip_mtime")
|
||||
filename = self.feed_options.get("gzip_filename")
|
||||
self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
|
||||
mtime=mtime, filename=filename)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.gzipfile.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.gzipfile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class Bz2Plugin:
|
||||
"""
|
||||
Compresses received data using `bz2 <https://en.wikipedia.org/wiki/Bzip2>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `bz2_compresslevel`
|
||||
|
||||
See :py:class:`bz2.BZ2File` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
compress_level = self.feed_options.get("bz2_compresslevel", 9)
|
||||
self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.bz2file.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.bz2file.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
class LZMAPlugin:
|
||||
"""
|
||||
Compresses received data using `lzma <https://en.wikipedia.org/wiki/Lempel–Ziv–Markov_chain_algorithm>`_.
|
||||
|
||||
Accepted ``feed_options`` parameters:
|
||||
|
||||
- `lzma_format`
|
||||
- `lzma_check`
|
||||
- `lzma_preset`
|
||||
- `lzma_filters`
|
||||
|
||||
.. note::
|
||||
``lzma_filters`` cannot be used in pypy version 7.3.1 and older.
|
||||
|
||||
See :py:class:`lzma.LZMAFile` for more info about parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
|
||||
format = self.feed_options.get("lzma_format")
|
||||
check = self.feed_options.get("lzma_check", -1)
|
||||
preset = self.feed_options.get("lzma_preset")
|
||||
filters = self.feed_options.get("lzma_filters")
|
||||
self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
|
||||
check=check, preset=preset, filters=filters)
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
return self.lzmafile.write(data)
|
||||
|
||||
def close(self) -> None:
|
||||
self.lzmafile.close()
|
||||
self.file.close()
|
||||
|
||||
|
||||
# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
|
||||
# instance as a file like writable object. This could be needed by some exporters
|
||||
# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper.
|
||||
class PostProcessingManager(IOBase):
|
||||
"""
|
||||
This will manage and use declared plugins to process data in a
|
||||
pipeline-ish way.
|
||||
:param plugins: all the declared plugins for the feed
|
||||
:type plugins: list
|
||||
:param file: final target file where the processed data will be written
|
||||
:type file: file like object
|
||||
"""
|
||||
|
||||
def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
|
||||
self.plugins = self._load_plugins(plugins)
|
||||
self.file = file
|
||||
self.feed_options = feed_options
|
||||
self.head_plugin = self._get_head_plugin()
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
"""
|
||||
Uses all the declared plugins to process data first, then writes
|
||||
the processed data to target file.
|
||||
:param data: data passed to be written to target file
|
||||
:type data: bytes
|
||||
:return: returns number of bytes written
|
||||
:rtype: int
|
||||
"""
|
||||
return self.head_plugin.write(data)
|
||||
|
||||
def tell(self) -> int:
|
||||
return self.file.tell()
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the target file along with all the plugins.
|
||||
"""
|
||||
self.head_plugin.close()
|
||||
|
||||
def writable(self) -> bool:
|
||||
return True
|
||||
|
||||
def _load_plugins(self, plugins: List[Any]) -> List[Any]:
|
||||
plugins = [load_object(plugin) for plugin in plugins]
|
||||
return plugins
|
||||
|
||||
def _get_head_plugin(self) -> Any:
|
||||
prev = self.file
|
||||
for plugin in self.plugins[::-1]:
|
||||
prev = plugin(prev, self.feed_options)
|
||||
return prev
|
||||
|
|
@ -8,6 +8,7 @@ from scrapy import signals
|
|||
from scrapy.mail import MailSender
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
||||
class StatsMailer:
|
||||
|
||||
def __init__(self, stats, recipients, mail):
|
||||
|
|
|
|||
|
|
@ -142,10 +142,6 @@ class WrappedRequest:
|
|||
"""
|
||||
return self.request.meta.get('is_unverifiable', False)
|
||||
|
||||
def get_origin_req_host(self):
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
# python3 uses attributes instead of methods
|
||||
@property
|
||||
def full_url(self):
|
||||
return self.get_full_url()
|
||||
|
|
@ -164,7 +160,7 @@ class WrappedRequest:
|
|||
|
||||
@property
|
||||
def origin_req_host(self):
|
||||
return self.get_origin_req_host()
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
def has_header(self, name):
|
||||
return name in self.request.headers
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
|
|||
"""
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from typing import Generator, Tuple
|
||||
from urllib.parse import urljoin
|
||||
|
|
@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
http_content_type_encoding, resolve_encoding)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
|
@ -66,13 +64,6 @@ class TextResponse(Response):
|
|||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
warnings.warn('Response.body_as_unicode() is deprecated, '
|
||||
'please use Response.text instead.',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.text
|
||||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
|
|
|||
|
|
@ -9,45 +9,15 @@ from collections.abc import MutableMapping
|
|||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from typing import Dict
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
class _BaseItem(object_ref):
|
||||
"""
|
||||
Temporary class used internally to avoid the deprecation
|
||||
warning raised by isinstance checks using BaseItem.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class _BaseItemMeta(ABCMeta):
|
||||
def __instancecheck__(cls, instance):
|
||||
if cls is BaseItem:
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
|
||||
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
||||
"""
|
||||
Deprecated, please use :class:`scrapy.item.Item` instead
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
class ItemMeta(_BaseItemMeta):
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
.. _metaclass: https://realpython.com/python-metaclasses
|
||||
|
|
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
|
|||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
fields: Dict[str, Field] = {}
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
||||
fields: Dict[str, Field]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __iter__(self):
|
||||
return iter(self._values)
|
||||
|
||||
__hash__ = BaseItem.__hash__
|
||||
__hash__ = object_ref.__hash__
|
||||
|
||||
def keys(self):
|
||||
return self._values.keys()
|
||||
|
|
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
|
|||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class LxmlParserLinkExtractor:
|
|||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
||||
The subclass should override it if neccessary
|
||||
The subclass should override it if necessary
|
||||
"""
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader):
|
|||
|
||||
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
|
||||
if selector is None and response is not None:
|
||||
selector = self.default_selector_class(response)
|
||||
try:
|
||||
selector = self.default_selector_class(response)
|
||||
except AttributeError:
|
||||
selector = None
|
||||
context.update(response=response)
|
||||
super().__init__(item=item, selector=selector, parent=parent, **context)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import logging
|
||||
import pprint
|
||||
from collections import defaultdict, deque
|
||||
from typing import Callable, Deque, Dict
|
||||
from typing import Callable, Deque, Dict, Optional, cast, Iterable
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -9,7 +9,7 @@ from scrapy import Spider
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
|
||||
from scrapy.utils.defer import process_parallel, process_chain
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -21,7 +21,8 @@ class MiddlewareManager:
|
|||
|
||||
def __init__(self, *middlewares):
|
||||
self.middlewares = middlewares
|
||||
self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
|
||||
# Optional because process_spider_output and process_spider_exception can be None
|
||||
self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
|
||||
for mw in middlewares:
|
||||
self._add_middleware(mw)
|
||||
|
||||
|
|
@ -64,14 +65,12 @@ class MiddlewareManager:
|
|||
self.methods['close_spider'].appendleft(mw.close_spider)
|
||||
|
||||
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_parallel(self.methods[methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_parallel(methods, obj, *args)
|
||||
|
||||
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain(self.methods[methodname], obj, *args)
|
||||
|
||||
def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain_both(self.methods[cb_methodname],
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_chain(methods, obj, *args)
|
||||
|
||||
def open_spider(self, spider: Spider) -> Deferred:
|
||||
return self._process_parallel('open_spider', spider)
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class S3FilesStore:
|
|||
AWS_USE_SSL = None
|
||||
AWS_VERIFY = None
|
||||
|
||||
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
||||
POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
||||
HEADERS = {
|
||||
'Cache-Control': 'max-age=172800',
|
||||
}
|
||||
|
|
@ -142,7 +142,7 @@ class S3FilesStore:
|
|||
**extra)
|
||||
|
||||
def _headers_to_botocore_kwargs(self, headers):
|
||||
""" Convert headers to botocore keyword agruments.
|
||||
""" Convert headers to botocore keyword arguments.
|
||||
"""
|
||||
# This is required while we need to support both boto and botocore.
|
||||
mapping = CaseInsensitiveDict({
|
||||
|
|
@ -190,7 +190,7 @@ class GCSFilesStore:
|
|||
CACHE_CONTROL = 'max-age=172800'
|
||||
|
||||
# The bucket's default object ACL will be applied to the object.
|
||||
# Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
||||
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
|
||||
POLICY = None
|
||||
|
||||
def __init__(self, uri):
|
||||
|
|
@ -291,7 +291,7 @@ class FilesPipeline(MediaPipeline):
|
|||
"""Abstract pipeline that implement the file downloading
|
||||
|
||||
This pipeline tries to minimize network transfers and file processing,
|
||||
doing stat of the files and determining if file is new, uptodate or
|
||||
doing stat of the files and determining if file is new, up-to-date or
|
||||
expired.
|
||||
|
||||
``new`` files are those that pipeline never processed and needs to be
|
||||
|
|
|
|||
|
|
@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
|
|||
return len(self.attributes)
|
||||
|
||||
def _to_dict(self):
|
||||
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
for k, v in self.items()}
|
||||
|
||||
def _get_key(self, key_value):
|
||||
return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
|
||||
else str(key_value))
|
||||
|
||||
def copy_to_dict(self):
|
||||
"""
|
||||
Make a copy of current settings and convert to a dict.
|
||||
|
|
|
|||
|
|
@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
|
|||
LOG_STDOUT = False
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
LOG_FILE = None
|
||||
LOG_FILE_APPEND = True
|
||||
LOG_SHORT_NAMES = False
|
||||
|
||||
SCHEDULER_DEBUG = False
|
||||
|
|
|
|||
|
|
@ -4,14 +4,12 @@ Base class for Scrapy spiders
|
|||
See documentation in docs/topics/spiders.rst
|
||||
"""
|
||||
import logging
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
from scrapy.utils.deprecate import method_is_overridden
|
||||
|
||||
|
||||
class Spider(object_ref):
|
||||
|
|
@ -57,34 +55,13 @@ class Spider(object_ref):
|
|||
crawler.signals.connect(self.close, signals.spider_closed)
|
||||
|
||||
def start_requests(self):
|
||||
cls = self.__class__
|
||||
if not self.start_urls and hasattr(self, 'start_url'):
|
||||
raise AttributeError(
|
||||
"Crawling could not start: 'start_urls' not found "
|
||||
"or empty (but found 'start_url' attribute instead, "
|
||||
"did you miss an 's'?)")
|
||||
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
|
||||
warnings.warn(
|
||||
"Spider.make_requests_from_url method is deprecated; it "
|
||||
"won't be called in future Scrapy releases. Please "
|
||||
"override Spider.start_requests method instead "
|
||||
f"(see {cls.__module__}.{cls.__name__}).",
|
||||
)
|
||||
for url in self.start_urls:
|
||||
yield self.make_requests_from_url(url)
|
||||
else:
|
||||
for url in self.start_urls:
|
||||
yield Request(url, dont_filter=True)
|
||||
|
||||
def make_requests_from_url(self, url):
|
||||
""" This method is deprecated. """
|
||||
warnings.warn(
|
||||
"Spider.make_requests_from_url method is deprecated: "
|
||||
"it will be removed and not be called by the default "
|
||||
"Spider.start_requests method in future Scrapy releases. "
|
||||
"Please override Spider.start_requests method instead."
|
||||
)
|
||||
return Request(url, dont_filter=True)
|
||||
for url in self.start_urls:
|
||||
yield Request(url, dont_filter=True)
|
||||
|
||||
def _parse(self, response, **kwargs):
|
||||
return self.parse(response, **kwargs)
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_node(self, response, selector):
|
||||
"""This method must be overriden with your custom spider functionality"""
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
if hasattr(self, 'parse_item'): # backward compatibility
|
||||
return self.parse_item(response, selector)
|
||||
raise NotImplementedError
|
||||
|
|
@ -113,7 +113,7 @@ class CSVFeedSpider(Spider):
|
|||
return response
|
||||
|
||||
def parse_row(self, response, row):
|
||||
"""This method must be overriden with your custom spider functionality"""
|
||||
"""This method must be overridden with your custom spider functionality"""
|
||||
raise NotImplementedError
|
||||
|
||||
def parse_rows(self, response):
|
||||
|
|
@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
|
|||
process_results methods for pre and post-processing purposes.
|
||||
"""
|
||||
|
||||
for row in csviter(response, self.delimiter, self.headers, self.quotechar):
|
||||
for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
|
||||
ret = iterate_spider_output(self.parse_row(response, row))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
|
|||
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
|
||||
out.setdefault("item_export_kwargs", dict())
|
||||
out.setdefault("item_export_kwargs", {})
|
||||
if settings["FEED_EXPORT_INDENT"] is None:
|
||||
out.setdefault("indent", None)
|
||||
else:
|
||||
|
|
@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
|
|||
message = (
|
||||
'The -t command line option is deprecated in favor of '
|
||||
'specifying the output format within the output URI. See the '
|
||||
'documentation of the -o and -O options for more information.',
|
||||
'documentation of the -o and -O options for more information.'
|
||||
)
|
||||
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
|
||||
return {output[0]: {'format': output_format}}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
|
|||
@wraps(_embed_ipython_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
config = load_default_config()
|
||||
# Always use .instace() to ensure _instance propagation to all parents
|
||||
# Always use .instance() to ensure _instance propagation to all parents
|
||||
# this is needed for <TAB> completion works well for new imports
|
||||
# and clear the instance to always have the fresh env
|
||||
# on repeated breaks like with inspect_response()
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class CaselessDict(dict):
|
|||
return key.lower()
|
||||
|
||||
def normvalue(self, value):
|
||||
"""Method to normalize values prior to be setted"""
|
||||
"""Method to normalize values prior to be set"""
|
||||
return value
|
||||
|
||||
def get(self, key, def_val=None):
|
||||
|
|
|
|||
|
|
@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds
|
|||
"""
|
||||
import asyncio
|
||||
import inspect
|
||||
from collections.abc import Coroutine
|
||||
from asyncio import Future
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Generator, Iterable
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Coroutine,
|
||||
Generator,
|
||||
Iterable,
|
||||
Union
|
||||
)
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
|
||||
|
|
@ -34,7 +41,7 @@ def defer_succeed(result) -> Deferred:
|
|||
"""Same as twisted.internet.defer.succeed but delay calling callback until
|
||||
next reactor loop
|
||||
|
||||
It delays by 100ms so reactor has a chance to go trough readers and writers
|
||||
It delays by 100ms so reactor has a chance to go through readers and writers
|
||||
before attending pending delayed calls, so do not set delay to zero.
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
|
|||
return defer.fail(result)
|
||||
else:
|
||||
return defer.succeed(result)
|
||||
|
||||
|
||||
def deferred_to_future(d: Deferred) -> Future:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return an :class:`asyncio.Future` object that wraps *d*.
|
||||
|
||||
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
|
||||
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
|
||||
callables defined as coroutines <coroutine-support>`, you can only await on
|
||||
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
|
||||
allows you to wait on them::
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
additional_response = await deferred_to_future(d)
|
||||
"""
|
||||
return d.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
|
||||
"""
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return *d* as an object that can be awaited from a :ref:`Scrapy callable
|
||||
defined as a coroutine <coroutine-support>`.
|
||||
|
||||
What you can await in Scrapy callables defined as coroutines depends on the
|
||||
value of :setting:`TWISTED_REACTOR`:
|
||||
|
||||
- When not using the asyncio reactor, you can only await on
|
||||
:class:`~twisted.internet.defer.Deferred` objects.
|
||||
|
||||
- When :ref:`using the asyncio reactor <install-asyncio>`, you can only
|
||||
await on :class:`asyncio.Future` objects.
|
||||
|
||||
If you want to write code that uses ``Deferred`` objects but works with any
|
||||
reactor, use this function on all ``Deferred`` objects::
|
||||
|
||||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
extra_response = await maybe_deferred_to_future(d)
|
||||
"""
|
||||
if not is_asyncio_reactor_installed():
|
||||
return d
|
||||
else:
|
||||
return deferred_to_future(d)
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ def create_deprecated_class(
|
|||
# for implementation details
|
||||
def __instancecheck__(cls, inst):
|
||||
return any(cls.__subclasscheck__(c)
|
||||
for c in {type(inst), inst.__class__})
|
||||
for c in (type(inst), inst.__class__))
|
||||
|
||||
def __subclasscheck__(cls, sub):
|
||||
if cls is not DeprecatedClass.deprecated_class:
|
||||
|
|
|
|||
|
|
@ -124,8 +124,9 @@ def _get_handler(settings):
|
|||
""" Return a log handler object according to settings """
|
||||
filename = settings.get('LOG_FILE')
|
||||
if filename:
|
||||
mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
|
||||
encoding = settings.get('LOG_ENCODING')
|
||||
handler = logging.FileHandler(filename, encoding=encoding)
|
||||
handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
|
||||
elif settings.getbool('LOG_ENABLED'):
|
||||
handler = logging.StreamHandler()
|
||||
else:
|
||||
|
|
@ -142,7 +143,7 @@ def _get_handler(settings):
|
|||
return handler
|
||||
|
||||
|
||||
def log_scrapy_info(settings):
|
||||
def log_scrapy_info(settings: Settings) -> None:
|
||||
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
|
||||
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
|
||||
versions = [
|
||||
|
|
@ -151,6 +152,9 @@ def log_scrapy_info(settings):
|
|||
if name != "Scrapy"
|
||||
]
|
||||
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
|
||||
|
||||
|
||||
def log_reactor_info() -> None:
|
||||
from twisted.internet import reactor
|
||||
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
|
||||
from twisted.internet import asyncioreactor
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ from w3lib.html import replace_entities
|
|||
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
|
||||
|
||||
|
||||
def arg_to_iter(arg):
|
||||
|
|
@ -50,7 +50,7 @@ def load_object(path):
|
|||
return path
|
||||
else:
|
||||
raise TypeError("Unexpected argument type, expected string "
|
||||
"or object, got: %s" % type(path))
|
||||
f"or object, got: {type(path)}")
|
||||
|
||||
try:
|
||||
dot = path.rindex('.')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import sys
|
||||
from contextlib import suppress
|
||||
|
||||
from twisted.internet import asyncioreactor, error
|
||||
|
|
@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None):
|
|||
reactor_class = load_object(reactor_path)
|
||||
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
|
||||
with suppress(error.ReactorAlreadyInstalledError):
|
||||
if sys.version_info >= (3, 8) and sys.platform == "win32":
|
||||
policy = asyncio.get_event_loop_policy()
|
||||
if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
if event_loop_path is not None:
|
||||
event_loop_class = load_object(event_loop_path)
|
||||
event_loop = event_loop_class()
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def request_fingerprint(
|
|||
the fingerprint.
|
||||
|
||||
For this reason, request headers are ignored by default when calculating
|
||||
the fingeprint. If you want to include specific headers use the
|
||||
the fingerprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
|
|
@ -78,7 +78,7 @@ def request_fingerprint(
|
|||
|
||||
|
||||
def request_authenticate(request: Request, username: str, password: str) -> None:
|
||||
"""Autenticate the given request (in place) using the HTTP basic access
|
||||
"""Authenticate the given request (in place) using the HTTP basic access
|
||||
authentication mechanism (RFC 2617) and the given username and password
|
||||
"""
|
||||
request.headers['Authorization'] = basic_auth_header(username, password)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ This module provides some useful functions for working with
|
|||
scrapy.http.Response objects
|
||||
"""
|
||||
import os
|
||||
import webbrowser
|
||||
import re
|
||||
import tempfile
|
||||
import webbrowser
|
||||
from typing import Any, Callable, Iterable, Optional, Tuple, Union
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
|
|
@ -13,6 +14,7 @@ from scrapy.http.response import Response
|
|||
|
||||
from twisted.web import http
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from w3lib import html
|
||||
|
||||
|
||||
|
|
@ -50,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
|
|||
return f'{status_int} {to_unicode(message)}'
|
||||
|
||||
|
||||
@deprecated
|
||||
def response_httprepr(response: Response) -> bytes:
|
||||
"""Return raw HTTP representation (as bytes) of the given response. This
|
||||
is provided only for reference, since it's not the exact stream of bytes
|
||||
|
|
@ -80,8 +83,9 @@ def open_in_browser(
|
|||
body = response.body
|
||||
if isinstance(response, HtmlResponse):
|
||||
if b'<base' not in body:
|
||||
repl = f'<head><base href="{response.url}">'
|
||||
body = body.replace(b'<head>', to_bytes(repl))
|
||||
repl = fr'\1<base href="{response.url}">'
|
||||
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
|
||||
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
|
||||
ext = '.html'
|
||||
elif isinstance(response, TextResponse):
|
||||
ext = '.txt'
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ ItemForm
|
|||
ia['width'] = x.x('//p[@class="width"]')
|
||||
ia['volume'] = x.x('//p[@class="volume"]')
|
||||
|
||||
# another example passing parametes on instance
|
||||
# another example passing parameters on instance
|
||||
ia = NewsForm(response, encoding='utf-8')
|
||||
ia['name'] = x.x('//p[@class="name"]')
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue