mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into wrar-asyncio-parse-asyncgen-proper-rebased
This commit is contained in:
Adrián Chaves
commit
cb3afcf0e7
|
|
@ -8,6 +8,7 @@ skips:
|
|||
- B311
|
||||
- B320
|
||||
- B321
|
||||
- B324
|
||||
- B402 # https://github.com/scrapy/scrapy/issues/4180
|
||||
- B403
|
||||
- B404
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.5.0
|
||||
current_version = 2.6.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: typing
|
||||
|
|
@ -38,8 +37,5 @@ jobs:
|
|||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -73,9 +72,6 @@ jobs:
|
|||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
.. image:: /artwork/scrapy-logo.jpg
|
||||
:width: 400px
|
||||
.. image:: https://scrapy.org/img/scrapylogo.png
|
||||
|
||||
======
|
||||
Scrapy
|
||||
|
|
|
|||
|
|
@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
|
|||
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
|
||||
|
||||
# Never documented before, and deprecated now.
|
||||
r'^scrapy\.item\.DictItem$',
|
||||
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
|
||||
|
||||
# Implementation detail of LxmlLinkExtractor
|
||||
|
|
@ -304,10 +303,14 @@ intersphinx_mapping = {
|
|||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"command": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
"reqmeta": "tooltip",
|
||||
"setting": "tooltip",
|
||||
"signal": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
|
|
|||
|
|
@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
|
|||
|
||||
See :ref:`topics-request-response-ref-request-callback-arguments`.
|
||||
|
||||
|
||||
Scrapy crashes with: ImportError: No module named win32api
|
||||
----------------------------------------------------------
|
||||
|
||||
You need to install `pywin32`_ because of `this Twisted bug`_.
|
||||
|
||||
.. _pywin32: https://sourceforge.net/projects/pywin32/
|
||||
.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
|
||||
|
||||
How can I simulate a user login in my spider?
|
||||
---------------------------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ testing.
|
|||
.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
|
||||
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
|
||||
|
||||
.. _getting-help:
|
||||
|
||||
Getting help
|
||||
============
|
||||
|
||||
|
|
@ -24,12 +26,14 @@ Having trouble? We'd like to help!
|
|||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||
* Ask a question in the `#scrapy IRC channel`_,
|
||||
* Report bugs with Scrapy in our `issue tracker`_.
|
||||
* Join the Discord community `Scrapy Discord`_.
|
||||
|
||||
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
|
||||
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
|
||||
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
|
||||
|
||||
|
||||
First steps
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Examples
|
|||
The best way to learn is with examples, and Scrapy is no exception. For this
|
||||
reason, there is an example Scrapy project named quotesbot_, that you can use to
|
||||
play and learn more about Scrapy. It contains two spiders for
|
||||
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
expressions.
|
||||
|
||||
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
|
|||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
* *(Recommended)* **Don't** use system Python. Install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
|
|
@ -234,8 +234,8 @@ For PyPy3, only Linux installation was tested.
|
|||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependencies will be built during installation.
|
||||
On macOS, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
On macOS, you are likely to face an issue with building the Cryptography
|
||||
dependency. The solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing Scrapy). Installing on Linux has no special
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
|
|||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
https://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
'https://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Scrapy Tutorial
|
|||
In this tutorial, we'll assume that Scrapy is already installed on your system.
|
||||
If that's not the case, see :ref:`intro-install`.
|
||||
|
||||
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
|
||||
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
|
||||
that lists quotes from famous authors.
|
||||
|
||||
This tutorial will walk you through these tasks:
|
||||
|
|
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
|
|||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
|
@ -143,9 +143,9 @@ similar to this::
|
|||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
|
@ -184,8 +184,8 @@ for your spider::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -207,7 +207,7 @@ Extracting data
|
|||
The best way to learn how to extract data with Scrapy is trying selectors
|
||||
using the :ref:`Scrapy shell <topics-shell>`. Run::
|
||||
|
||||
scrapy shell 'http://quotes.toscrape.com/page/1/'
|
||||
scrapy shell 'https://quotes.toscrape.com/page/1/'
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "http://quotes.toscrape.com/page/1/"
|
||||
scrapy shell "https://quotes.toscrape.com/page/1/"
|
||||
|
||||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 http://quotes.toscrape.com/page/1/>
|
||||
[s] request <GET https://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 https://quotes.toscrape.com/page/1/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
|
||||
[s] Useful shortcuts:
|
||||
|
|
@ -241,7 +241,7 @@ object:
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
|
@ -355,7 +355,7 @@ Extracting quotes and authors
|
|||
Now that you know a bit about selection and extraction, let's complete our
|
||||
spider by writing the code to extract the quotes from the web page.
|
||||
|
||||
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
|
||||
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
|
||||
like this:
|
||||
|
||||
.. code-block:: html
|
||||
|
|
@ -379,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'http://quotes.toscrape.com'
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
@ -444,8 +444,8 @@ in the callback, as you can see below::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -458,9 +458,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
@ -488,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily
|
|||
append new records to it. It doesn't have the same problem of JSON when you run
|
||||
twice. Also, as each record is a separate line, you can process big files
|
||||
without having to fit everything in memory, there are tools like `JQ`_ to help
|
||||
doing that at the command-line.
|
||||
do that at the command-line.
|
||||
|
||||
In small projects (like the one in this tutorial), that should be enough.
|
||||
However, if you want to perform more complex things with the scraped items, you
|
||||
|
|
@ -505,7 +505,7 @@ Following links
|
|||
===============
|
||||
|
||||
Let's say, instead of just scraping the stuff from the first two pages
|
||||
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
|
||||
Now that you know how to extract data from pages, let's see how to follow links
|
||||
from them.
|
||||
|
|
@ -549,7 +549,7 @@ page, extracting data from it::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -654,7 +654,7 @@ this time for scraping author information::
|
|||
class AuthorSpider(scrapy.Spider):
|
||||
name = 'author'
|
||||
|
||||
start_urls = ['http://quotes.toscrape.com/']
|
||||
start_urls = ['https://quotes.toscrape.com/']
|
||||
|
||||
def parse(self, response):
|
||||
author_page_links = response.css('.author + a')
|
||||
|
|
@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument::
|
|||
name = "quotes"
|
||||
|
||||
def start_requests(self):
|
||||
url = 'http://quotes.toscrape.com/'
|
||||
url = 'https://quotes.toscrape.com/'
|
||||
tag = getattr(self, 'tag', None)
|
||||
if tag is not None:
|
||||
url = url + 'tag/' + tag
|
||||
|
|
@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument::
|
|||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
will only visit URLs from the ``humor`` tag, such as
|
||||
``http://quotes.toscrape.com/tag/humor``.
|
||||
``https://quotes.toscrape.com/tag/humor``.
|
||||
|
||||
You can :ref:`learn more about handling spider arguments here <spiderargs>`.
|
||||
|
||||
|
|
|
|||
479
docs/news.rst
479
docs/news.rst
|
|
@ -1,30 +1,414 @@
|
|||
.. note::
|
||||
.. versionchanged:: VERSION
|
||||
|
||||
The Twisted reactor is now installed when
|
||||
:meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
|
||||
:class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
|
||||
:setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
|
||||
honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
|
||||
they are silently ignored when set there and you need to set these settings
|
||||
in some other way.
|
||||
|
||||
|
||||
.. note::
|
||||
.. versionchanged:: VERSION
|
||||
|
||||
Previously this setting had no effect in a spider
|
||||
:attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
|
||||
if you :ref:`run several spiders in one process <run-multiple-spiders>`,
|
||||
they must not have different values for this setting, because they will use
|
||||
a single reactor instance.
|
||||
|
||||
|
||||
.. _news:
|
||||
|
||||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.6.1:
|
||||
|
||||
Scrapy 2.6.1 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Fixes a regression introduced in 2.6.0 that would unset the request method when
|
||||
following redirects.
|
||||
|
||||
|
||||
.. _release-2.6.0:
|
||||
|
||||
Scrapy 2.6.0 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Security fixes for cookie handling <2.6-security-fixes>`
|
||||
|
||||
* Python 3.10 support
|
||||
|
||||
* :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental, and works out-of-the-box on Windows regardless of your Python
|
||||
version
|
||||
|
||||
* Feed exports now support :class:`pathlib.Path` output paths and per-feed
|
||||
:ref:`item filtering <item-filter>` and
|
||||
:ref:`post-processing <post-processing>`
|
||||
|
||||
.. _2.6-security-fixes:
|
||||
|
||||
Security bug fixes
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies from a
|
||||
controlled domain into your cookiejar that could be sent to other domains
|
||||
not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
|
||||
advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The h2_ dependency is now optional, only needed to
|
||||
:ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
|
||||
|
||||
.. _h2: https://pypi.org/project/h2/
|
||||
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
|
||||
for a non-POST request, now overrides the URL query string, instead of
|
||||
being appended to it. (:issue:`2919`, :issue:`3579`)
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
|
||||
the return value of that function, and not the ``params`` input parameter,
|
||||
will determine the feed URI parameters, unless that return value is
|
||||
``None``. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- In :class:`scrapy.core.engine.ExecutionEngine`, methods
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download`,
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
|
||||
and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
now raise :exc:`RuntimeError` if called before
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
|
||||
|
||||
These methods used to assume that
|
||||
:attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
|
||||
been defined by a prior call to
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
|
||||
raising :exc:`AttributeError` instead.
|
||||
|
||||
- If the API of the configured :ref:`scheduler <topics-scheduler>` does not
|
||||
meet expectations, :exc:`TypeError` is now raised at startup time. Before,
|
||||
other exceptions would be raised at run time. (:issue:`3559`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
|
||||
now been removed. (:issue:`5393`)
|
||||
|
||||
- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
|
||||
(:issue:`5398`)
|
||||
|
||||
- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
|
||||
been removed. (:issue:`4178`, :issue:`4356`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
|
||||
returning ``None`` or modifying the ``params`` input parameter is now
|
||||
deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
|
||||
:meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
|
||||
|
||||
- Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
|
||||
:func:`scrapy.utils.request.request_from_dict` function.
|
||||
|
||||
- In :mod:`scrapy.squeues`, the following queue classes are deprecated:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
|
||||
instead use:
|
||||
:class:`~scrapy.squeues.PickleFifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.PickleLifoDiskQueue`,
|
||||
:class:`~scrapy.squeues.MarshalFifoDiskQueue`,
|
||||
and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
|
||||
|
||||
- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
|
||||
a time when this class could handle multiple :class:`~scrapy.Spider`
|
||||
objects at a time have been deprecated. (:issue:`5090`)
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
|
||||
is deprecated.
|
||||
|
||||
- The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
|
||||
deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
|
||||
:meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
|
||||
|
||||
- The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
|
||||
is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
|
||||
instead.
|
||||
|
||||
- The ``spider`` parameter is deprecated for the following methods:
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
|
||||
|
||||
- :meth:`~scrapy.core.engine.ExecutionEngine.download`
|
||||
|
||||
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
|
||||
first to set the :class:`~scrapy.Spider` object.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- You can now use :ref:`item filtering <item-filter>` to control which items
|
||||
are exported to each output feed. (:issue:`4575`, :issue:`5178`,
|
||||
:issue:`5161`, :issue:`5203`)
|
||||
|
||||
- You can now apply :ref:`post-processing <post-processing>` to feeds, and
|
||||
:ref:`built-in post-processing plugins <builtin-plugins>` are provided for
|
||||
output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
|
||||
|
||||
- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
|
||||
keys. (:issue:`5383`, :issue:`5384`)
|
||||
|
||||
- Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
|
||||
or later will automatically switch the asyncio event loop to one that
|
||||
allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
|
||||
:issue:`5315`)
|
||||
|
||||
- The :command:`genspider` command now supports a start URL instead of a
|
||||
domain name. (:issue:`4439`)
|
||||
|
||||
- :mod:`scrapy.utils.defer` gained 2 new functions,
|
||||
:func:`~scrapy.utils.defer.deferred_to_future` and
|
||||
:func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
|
||||
on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
|
||||
(:issue:`5288`)
|
||||
|
||||
- :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
|
||||
support for `temporary security credentials`_
|
||||
(:setting:`AWS_SESSION_TOKEN`) and endpoint customization
|
||||
(:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
|
||||
|
||||
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
|
||||
|
||||
- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
|
||||
(:issue:`5279`)
|
||||
|
||||
- :attr:`Request.cookies <scrapy.Request.cookies>` values that are
|
||||
:class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
|
||||
(:issue:`5252`, :issue:`5253`)
|
||||
|
||||
- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
|
||||
the :signal:`spider_idle` signal to customize the reason why the spider is
|
||||
stopping. (:issue:`5191`)
|
||||
|
||||
- When using
|
||||
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
|
||||
proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
|
||||
scheme. (:issue:`4505`, :issue:`4649`)
|
||||
|
||||
- All built-in queues now expose a ``peek`` method that returns the next
|
||||
queue object (like ``pop``) but does not remove the returned object from
|
||||
the queue. (:issue:`5112`)
|
||||
|
||||
If the underlying queue does not support peeking (e.g. because you are not
|
||||
using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
|
||||
:exc:`NotImplementedError`.
|
||||
|
||||
- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
|
||||
an ``attributes`` attribute that makes subclassing easier. For
|
||||
:class:`~scrapy.http.Request`, it also allows subclasses to work with
|
||||
:func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
|
||||
:issue:`5130`, :issue:`5218`)
|
||||
|
||||
- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
|
||||
:meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
|
||||
:ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
|
||||
|
||||
- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
|
||||
exceptions now only truncate response bodies longer than 1000 characters,
|
||||
instead of those longer than 32 characters, making it easier to debug such
|
||||
errors. (:issue:`4881`, :issue:`5007`)
|
||||
|
||||
- :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
|
||||
(:issue:`5145`, :issue:`5269`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
|
||||
are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
|
||||
(:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Removed a module-level Twisted reactor import that could prevent
|
||||
:ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
|
||||
|
||||
- The :command:`startproject` command works with existing folders again.
|
||||
(:issue:`4665`, :issue:`4676`)
|
||||
|
||||
- The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
|
||||
(:issue:`4962`, :issue:`4966`)
|
||||
|
||||
- :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
|
||||
``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
|
||||
|
||||
- Made :func:`scrapy.utils.response.open_in_browser` support more complex
|
||||
HTML. (:issue:`5319`, :issue:`5320`)
|
||||
|
||||
- Fixed :attr:`CSVFeedSpider.quotechar
|
||||
<scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
|
||||
encoding. (:issue:`5391`, :issue:`5394`)
|
||||
|
||||
- Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
|
||||
|
||||
.. _setuptools: https://pypi.org/project/setuptools/
|
||||
|
||||
- :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
now also works as expected with links that have comma-separated ``rel``
|
||||
attribute values including ``nofollow``. (:issue:`5225`)
|
||||
|
||||
- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
|
||||
<topics-feed-exports>` parameter parsing. (:issue:`5359`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- :ref:`asyncio support <using-asyncio>` is no longer considered
|
||||
experimental. (:issue:`5332`)
|
||||
|
||||
- Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
|
||||
(:issue:`4976`, :issue:`5315`)
|
||||
|
||||
- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
|
||||
(:issue:`4484`, :issue:`4613`)
|
||||
|
||||
- Documented :ref:`local file naming in media pipelines
|
||||
<topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
|
||||
|
||||
- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
|
||||
:issue:`3669`)
|
||||
|
||||
- Provided better context and instructions to disable the
|
||||
:setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
|
||||
|
||||
- Documented that :ref:`reppy-parser` does not support Python 3.9+.
|
||||
(:issue:`5226`, :issue:`5231`)
|
||||
|
||||
- Documented :ref:`the scheduler component <topics-scheduler>`.
|
||||
(:issue:`3537`, :issue:`3559`)
|
||||
|
||||
- Documented the method used by :ref:`media pipelines
|
||||
<topics-media-pipeline>` to :ref:`determine if a file has expired
|
||||
<file-expiration>`. (:issue:`5120`, :issue:`5254`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now features
|
||||
:func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
|
||||
|
||||
- :ref:`run-multiple-spiders` now covers what happens when you define
|
||||
different per-spider values for some settings that cannot differ at run
|
||||
time. (:issue:`4485`, :issue:`5352`)
|
||||
|
||||
- Extended the documentation of the
|
||||
:class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
|
||||
(:issue:`5199`, :issue:`5217`)
|
||||
|
||||
- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
|
||||
:issue:`5224`)
|
||||
|
||||
- Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
|
||||
(:issue:`5174`, :issue:`5244`)
|
||||
|
||||
- Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
|
||||
(:issue:`1582`)
|
||||
|
||||
- Added the ``body_length`` parameter to the documented signature of the
|
||||
:signal:`headers_received` signal. (:issue:`5270`)
|
||||
|
||||
- Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
|
||||
in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
|
||||
|
||||
- The documentation now features the shortest import path of classes with
|
||||
multiple import paths. (:issue:`2733`, :issue:`5099`)
|
||||
|
||||
- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
|
||||
(:issue:`5395`, :issue:`5396`)
|
||||
|
||||
- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
|
||||
to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
|
||||
|
||||
- The pronunciation of the project name is now :ref:`officially
|
||||
<intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
|
||||
|
||||
- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
|
||||
|
||||
- Fixed issues and implemented minor improvements. (:issue:`3155`,
|
||||
:issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
|
||||
:issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
|
||||
:issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
|
||||
:issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
|
||||
|
||||
|
||||
Quality Assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
|
||||
:issue:`5265`)
|
||||
|
||||
- Significantly reduced memory usage by
|
||||
:func:`scrapy.utils.response.response_httprepr`, used by the
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
|
||||
middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
|
||||
|
||||
- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
|
||||
:issue:`5374`)
|
||||
|
||||
- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
|
||||
:issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
|
||||
|
||||
- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
|
||||
:issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
|
||||
:issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
|
||||
:issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
|
||||
:issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
|
||||
:issue:`5425`, :issue:`5427`)
|
||||
|
||||
- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
|
||||
:issue:`5177`, :issue:`5200`)
|
||||
|
||||
- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
|
||||
:issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
|
||||
:issue:`5314`, :issue:`5322`)
|
||||
|
||||
|
||||
.. _release-2.5.1:
|
||||
|
||||
Scrapy 2.5.1 (2021-10-05)
|
||||
|
|
@ -1000,9 +1384,8 @@ Bug fixes
|
|||
* zope.interface 5.0.0 and later versions are now supported
|
||||
(:issue:`4447`, :issue:`4448`)
|
||||
|
||||
* :meth:`Spider.make_requests_from_url
|
||||
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
|
||||
1.4.0, now issues a warning when used (:issue:`4412`)
|
||||
* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
|
||||
warning when used (:issue:`4412`)
|
||||
|
||||
|
||||
Documentation
|
||||
|
|
@ -1514,6 +1897,50 @@ affect subclasses:
|
|||
(:issue:`3884`)
|
||||
|
||||
|
||||
.. _release-1.8.2:
|
||||
|
||||
Scrapy 1.8.2 (2022-03-01)
|
||||
-------------------------
|
||||
|
||||
**Security bug fixes:**
|
||||
|
||||
- When a :class:`~scrapy.http.Request` object with cookies defined gets a
|
||||
redirect response causing a new :class:`~scrapy.http.Request` object to be
|
||||
scheduled, the cookies defined in the original
|
||||
:class:`~scrapy.http.Request` object are no longer copied into the new
|
||||
:class:`~scrapy.http.Request` object.
|
||||
|
||||
If you manually set the ``Cookie`` header on a
|
||||
:class:`~scrapy.http.Request` object and the domain name of the redirect
|
||||
URL is not an exact match for the domain of the URL of the original
|
||||
:class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
|
||||
from the new :class:`~scrapy.http.Request` object.
|
||||
|
||||
The old behavior could be exploited by an attacker to gain access to your
|
||||
cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
|
||||
information.
|
||||
|
||||
.. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
|
||||
|
||||
.. note:: It is still possible to enable the sharing of cookies between
|
||||
different domains with a shared domain suffix (e.g.
|
||||
``example.com`` and any subdomain) by defining the shared domain
|
||||
suffix (e.g. ``example.com``) as the cookie domain when defining
|
||||
your cookies. See the documentation of the
|
||||
:class:`~scrapy.http.Request` class for more information.
|
||||
|
||||
- When the domain of a cookie, either received in the ``Set-Cookie`` header
|
||||
of a response or defined in a :class:`~scrapy.http.Request` object, is set
|
||||
to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
|
||||
ignored unless the cookie domain is the same as the request domain.
|
||||
|
||||
The old behavior could be exploited by an attacker to inject cookies into
|
||||
your requests to some other domains. Please, see the `mfjm-vh54-3f96
|
||||
security advisory`_ for more information.
|
||||
|
||||
.. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
|
||||
|
||||
|
||||
.. _release-1.8.1:
|
||||
|
||||
Scrapy 1.8.1 (2021-10-05)
|
||||
|
|
@ -4380,7 +4807,7 @@ Code rearranged and removed
|
|||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
|||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
Installing the asyncio reactor
|
||||
|
|
@ -25,6 +26,7 @@ reactor manually. You can do that using
|
|||
|
||||
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
|
||||
|
||||
|
||||
.. _using-custom-loops:
|
||||
|
||||
Using custom asyncio loops
|
||||
|
|
@ -34,20 +36,30 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
|
|||
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
|
||||
use it instead of the default asyncio event loop.
|
||||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
.. _asyncio-windows:
|
||||
|
||||
Windows-specific notes
|
||||
======================
|
||||
|
||||
The Windows implementation of :mod:`asyncio` can use two event loop
|
||||
implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
|
||||
3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
|
||||
(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
|
||||
event loop class needs to be changed. Scrapy since VERSION does this
|
||||
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
|
||||
by other means or use an older Scrapy version you need to call the following
|
||||
code before installing the reactor::
|
||||
implementations:
|
||||
|
||||
- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
|
||||
when using Twisted.
|
||||
|
||||
- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
|
||||
with Twisted.
|
||||
|
||||
So on Python 3.8+ the event loop class needs to be changed.
|
||||
|
||||
.. versionchanged:: 2.6.0
|
||||
The event loop class is changed automatically when you change the
|
||||
:setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||
|
||||
To change the event loop class manually, call the following code before
|
||||
installing the reactor::
|
||||
|
||||
import asyncio
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
|
@ -64,6 +76,9 @@ yourself, or in some code that runs before the reactor is installed, e.g.
|
|||
|
||||
.. _playwright: https://github.com/microsoft/playwright-python
|
||||
|
||||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
Awaiting on Deferreds
|
||||
=====================
|
||||
|
||||
|
|
|
|||
|
|
@ -230,10 +230,16 @@ Usage example::
|
|||
genspider
|
||||
---------
|
||||
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain>``
|
||||
* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
|
||||
* Requires project: *no*
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
.. versionadded:: 2.6.0
|
||||
The ability to pass a URL instead of a domain.
|
||||
|
||||
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
|
||||
|
||||
.. note:: Even if an HTTPS URL is specified, the protocol used in
|
||||
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
|
|
|||
|
|
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
|
|||
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
|
||||
copy XPaths to selected elements. Let's try it out.
|
||||
|
||||
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
|
||||
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
$ scrapy shell "http://quotes.toscrape.com/"
|
||||
$ scrapy shell "https://quotes.toscrape.com/"
|
||||
|
||||
Then, back to your web browser, right-click on the ``span`` tag, select
|
||||
``Copy > XPath`` and paste it in the Scrapy shell like so:
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
|
||||
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
|
||||
|
||||
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
|
||||
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
|
||||
|
|
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
|
|||
type ``json``.
|
||||
|
||||
If we click on this request, we see that the request URL is
|
||||
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
is a JSON-object that contains our quotes. We can also right-click
|
||||
on the request and open ``Open in new tab`` to get a better overview.
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
|
|||
name = 'quote'
|
||||
allowed_domains = ['quotes.toscrape.com']
|
||||
page = 1
|
||||
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
|
||||
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
|
||||
|
||||
def parse(self, response):
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
|
|||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
self.page += 1
|
||||
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
This spider starts at the first page of the quotes-API. With each
|
||||
|
|
@ -280,7 +280,7 @@ request::
|
|||
from scrapy import Request
|
||||
|
||||
request = Request.from_curl(
|
||||
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
|
||||
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
|
||||
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
|
||||
|
|
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
|
|||
to identifying the correct request and replicating it in your spider.
|
||||
|
||||
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
|
||||
.. _quotes.toscrape.com: http://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _quotes.toscrape.com: https://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
methods of installed middleware is always called on every response.
|
||||
|
||||
If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
|
||||
process_request methods and reschedule the returned request. Once the newly returned
|
||||
:meth:`process_request` methods and reschedule the returned request. Once the newly returned
|
||||
request is performed, the appropriate middleware chain will be called on
|
||||
the downloaded response.
|
||||
|
||||
|
|
@ -1019,7 +1019,7 @@ Parsers vary in several aspects:
|
|||
(shorter) rule
|
||||
|
||||
Performance comparison of different parsers is available at `the following link
|
||||
<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
|
||||
<https://github.com/scrapy/scrapy/issues/3969>`_.
|
||||
|
||||
.. _protego-parser:
|
||||
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ feed URI, allowing item delivery to start way before the end of the crawl.
|
|||
Item filtering
|
||||
==============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
You can filter items that you want to allow for a particular feed by using the
|
||||
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
|
||||
|
|
@ -318,11 +318,11 @@ ItemFilter
|
|||
Post-Processing
|
||||
===============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
|
|
@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods:
|
|||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
|
|
@ -457,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
If undefined or empty, all items are exported.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
|
||||
|
||||
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
|
|
@ -499,7 +499,7 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
The plugins will be used in the order of the list passed.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -744,6 +744,9 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
|
|
|
|||
|
|
@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many
|
|||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
|
|
|
|||
|
|
@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
|||
Additional features
|
||||
===================
|
||||
|
||||
.. _file-expiration:
|
||||
|
||||
File expiration
|
||||
---------------
|
||||
|
||||
|
|
|
|||
|
|
@ -262,7 +262,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
|
||||
cookies to spot bot behaviour
|
||||
* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
|
||||
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
|
||||
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
||||
directly
|
||||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||
|
|
@ -277,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _Common Crawl: https://commoncrawl.org/
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
|
||||
|
|
|
|||
|
|
@ -110,6 +110,10 @@ Request objects
|
|||
:class:`Request.cookies <scrapy.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. versionadded:: 2.6.0
|
||||
Cookie values that are :class:`bool`, :class:`float` or :class:`int`
|
||||
are casted to :class:`str`.
|
||||
|
||||
:type cookies: dict or list
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
@ -950,6 +954,14 @@ TextResponse objects
|
|||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
.. method:: TextResponse.urljoin(url)
|
||||
|
||||
Constructs an absolute url by combining the Response's base url with
|
||||
a possible relative url. The base url shall be extracted from the
|
||||
``<base>`` tag, or just the Response's :attr:`url` if there is no such
|
||||
tag.
|
||||
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
--------------------
|
||||
|
|
|
|||
|
|
@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy::
|
|||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes::
|
|||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ Let's take an example::
|
|||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
start_urls = ['https://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are
|
|||
different kinds of default spiders bundled into Scrapy for different purposes.
|
||||
We will talk about those types here.
|
||||
|
||||
.. module:: scrapy.spiders
|
||||
:synopsis: Spiders base class, spider manager and spider middleware
|
||||
|
||||
.. _topics-spiders-ref:
|
||||
|
||||
scrapy.Spider
|
||||
=============
|
||||
|
||||
.. class:: scrapy.spiders.Spider()
|
||||
.. class:: scrapy.spiders.Spider
|
||||
.. class:: scrapy.Spider()
|
||||
|
||||
This is the simplest spider, and the one from which every other spider
|
||||
|
|
|
|||
|
|
@ -21,5 +21,3 @@ addopts =
|
|||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
filterwarnings=
|
||||
ignore::DeprecationWarning:twisted.web.test.test_webclient
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.5.0
|
||||
2.6.1
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import argparse
|
||||
import cProfile
|
||||
import inspect
|
||||
import pkg_resources
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
|
|
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
|
||||
parser.description = cmd.long_desc()
|
||||
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler='resolve',
|
||||
description=cmd.long_desc())
|
||||
settings.setdict(cmd.default_settings, priority='command')
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
cmd.crawler_process = CrawlerProcess(settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from optparse import OptionGroup
|
||||
import argparse
|
||||
from typing import Any, Dict
|
||||
|
||||
from twisted.python import failure
|
||||
|
|
@ -59,22 +59,20 @@ class ScrapyCommand:
|
|||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
group = parser.add_argument_group(title='Global Options')
|
||||
group.add_argument("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_argument("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_argument("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_argument("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
try:
|
||||
|
|
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_argument("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_argument("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
|
||||
class ScrapyHelpFormatter(argparse.HelpFormatter):
|
||||
"""
|
||||
Help Formatter for scrapy command line help messages.
|
||||
"""
|
||||
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
|
||||
super().__init__(prog, indent_increment=indent_increment,
|
||||
max_help_position=max_help_position, width=width)
|
||||
|
||||
def _join_parts(self, part_strings):
|
||||
parts = self.format_part_strings(part_strings)
|
||||
return super()._join_parts(parts)
|
||||
|
||||
def format_part_strings(self, part_strings):
|
||||
"""
|
||||
Underline and title case command line help message headers.
|
||||
"""
|
||||
if part_strings and part_strings[0].startswith("usage: "):
|
||||
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
|
||||
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
|
||||
for index in headings[::-1]:
|
||||
char = '-' if "Global Options" in part_strings[index] else '='
|
||||
part_strings[index] = part_strings[index][:-2].title()
|
||||
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
|
||||
part_strings.insert(index + 1, underline)
|
||||
return part_strings
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
parser.add_argument("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -44,16 +44,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_argument("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_argument("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
|
|||
|
|
@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
parser.add_argument("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_argument("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_argument("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_argument("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_argument("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
|
|||
|
|
@ -18,16 +18,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
parser.add_argument("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_argument("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_argument("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ class Command(fetch.Command):
|
|||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
parser.add_argument('--headers', help=argparse.SUPPRESS)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from tldextract import TLDExtract
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_split_domain = TLDExtract(include_psl_private_domains=True)
|
||||
|
||||
|
||||
def _is_public_domain(domain):
|
||||
parts = _split_domain(domain)
|
||||
return not parts.domain
|
||||
|
||||
|
||||
class CookiesMiddleware:
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
|
||||
|
|
@ -23,14 +34,29 @@ class CookiesMiddleware:
|
|||
raise NotConfigured
|
||||
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
|
||||
|
||||
def _process_cookies(self, cookies, *, jar, request):
|
||||
for cookie in cookies:
|
||||
cookie_domain = cookie.domain
|
||||
if cookie_domain.startswith('.'):
|
||||
cookie_domain = cookie_domain[1:]
|
||||
|
||||
request_domain = urlparse_cached(request).hostname.lower()
|
||||
|
||||
if cookie_domain and _is_public_domain(cookie_domain):
|
||||
if cookie_domain != request_domain:
|
||||
continue
|
||||
cookie.domain = request_domain
|
||||
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if request.meta.get('dont_merge_cookies', False):
|
||||
return
|
||||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
for cookie in self._get_request_cookies(jar, request):
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
# set Cookie header
|
||||
request.headers.pop('Cookie', None)
|
||||
|
|
@ -44,7 +70,9 @@ class CookiesMiddleware:
|
|||
# extract cookies from Set-Cookie and drop invalid/expired cookies
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
jar.extract_cookies(response, request)
|
||||
cookies = jar.make_cookies(response, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
self._debug_set_cookie(response, spider)
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse
|
|||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
|
||||
|
|
@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_redirect_request(source_request, *, url, **kwargs):
|
||||
redirect_request = source_request.replace(
|
||||
url=url,
|
||||
**kwargs,
|
||||
cookies=None,
|
||||
)
|
||||
if 'Cookie' in redirect_request.headers:
|
||||
source_request_netloc = urlparse_cached(source_request).netloc
|
||||
redirect_request_netloc = urlparse_cached(redirect_request).netloc
|
||||
if source_request_netloc != redirect_request_netloc:
|
||||
del redirect_request.headers['Cookie']
|
||||
return redirect_request
|
||||
|
||||
|
||||
class BaseRedirectMiddleware:
|
||||
|
||||
enabled_setting = 'REDIRECT_ENABLED'
|
||||
|
|
@ -47,10 +62,15 @@ class BaseRedirectMiddleware:
|
|||
raise IgnoreRequest("max redirections reached")
|
||||
|
||||
def _redirect_request_using_get(self, request, redirect_url):
|
||||
redirected = request.replace(url=redirect_url, method='GET', body='')
|
||||
redirected.headers.pop('Content-Type', None)
|
||||
redirected.headers.pop('Content-Length', None)
|
||||
return redirected
|
||||
redirect_request = _build_redirect_request(
|
||||
request,
|
||||
url=redirect_url,
|
||||
method='GET',
|
||||
body='',
|
||||
)
|
||||
redirect_request.headers.pop('Content-Type', None)
|
||||
redirect_request.headers.pop('Content-Length', None)
|
||||
return redirect_request
|
||||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
|
|
@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
if response.status in (301, 307, 308) or request.method == 'HEAD':
|
||||
redirected = request.replace(url=redirected_url)
|
||||
redirected = _build_redirect_request(request, url=redirected_url)
|
||||
return self._redirect(redirected, request, spider, response.status)
|
||||
|
||||
redirected = self._redirect_request_using_get(request, redirected_url)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,22 @@
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.python import global_object_name, to_bytes
|
||||
from scrapy.utils.request import request_httprepr
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
from twisted.web import http
|
||||
|
||||
|
||||
def get_header_size(headers):
|
||||
size = 0
|
||||
for key, value in headers.items():
|
||||
if isinstance(value, (list, tuple)):
|
||||
for v in value:
|
||||
size += len(b": ") + len(key) + len(v)
|
||||
return size + len(b'\r\n') * (len(headers.keys()) - 1)
|
||||
|
||||
|
||||
def get_status_size(response_status):
|
||||
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
|
||||
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
|
||||
|
||||
|
||||
class DownloaderStats:
|
||||
|
|
@ -24,7 +39,8 @@ class DownloaderStats:
|
|||
def process_response(self, request, response, spider):
|
||||
self.stats.inc_value('downloader/response_count', spider=spider)
|
||||
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
|
||||
reslen = len(response_httprepr(response))
|
||||
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
|
||||
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
|
||||
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
|
||||
|
|
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
return serializer(value)
|
||||
|
||||
def _serialize_value(self, value):
|
||||
if isinstance(value, _BaseItem):
|
||||
if isinstance(value, Item):
|
||||
return self.export_item(value)
|
||||
elif is_item(value):
|
||||
return dict(self._serialize_item(value))
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ import sys
|
|||
import warnings
|
||||
from datetime import datetime
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import implementer, Interface
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
|
|
@ -524,7 +524,12 @@ class FeedExporter:
|
|||
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
def _get_uri_params(
|
||||
self,
|
||||
spider: Spider,
|
||||
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
|
||||
slot: Optional[_FeedSlot] = None,
|
||||
) -> dict:
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
|
|
@ -532,9 +537,18 @@ class FeedExporter:
|
|||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
original_params = params.copy()
|
||||
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
|
||||
new_params = uripar_function(params, spider)
|
||||
if new_params is None or original_params != params:
|
||||
warnings.warn(
|
||||
'Modifying the params dictionary in-place in the function defined in '
|
||||
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
|
||||
'setting is deprecated. The function must return a new dictionary '
|
||||
'instead.',
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return new_params if new_params is not None else params
|
||||
|
||||
def _load_filter(self, feed_options):
|
||||
# load the item filter if declared else load the default filter class
|
||||
|
|
|
|||
|
|
@ -142,10 +142,6 @@ class WrappedRequest:
|
|||
"""
|
||||
return self.request.meta.get('is_unverifiable', False)
|
||||
|
||||
def get_origin_req_host(self):
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
# python3 uses attributes instead of methods
|
||||
@property
|
||||
def full_url(self):
|
||||
return self.get_full_url()
|
||||
|
|
@ -164,7 +160,7 @@ class WrappedRequest:
|
|||
|
||||
@property
|
||||
def origin_req_host(self):
|
||||
return self.get_origin_req_host()
|
||||
return urlparse_cached(self.request).hostname
|
||||
|
||||
def has_header(self, name):
|
||||
return name in self.request.headers
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
|
|||
"""
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from typing import Generator, Tuple
|
||||
from urllib.parse import urljoin
|
||||
|
|
@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
http_content_type_encoding, resolve_encoding)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
|
@ -66,13 +64,6 @@ class TextResponse(Response):
|
|||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
warnings.warn('Response.body_as_unicode() is deprecated, '
|
||||
'please use Response.text instead.',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.text
|
||||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
|
|
|||
|
|
@ -9,45 +9,15 @@ from collections.abc import MutableMapping
|
|||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from typing import Dict
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
class _BaseItem(object_ref):
|
||||
"""
|
||||
Temporary class used internally to avoid the deprecation
|
||||
warning raised by isinstance checks using BaseItem.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class _BaseItemMeta(ABCMeta):
|
||||
def __instancecheck__(cls, instance):
|
||||
if cls is BaseItem:
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
|
||||
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
||||
"""
|
||||
Deprecated, please use :class:`scrapy.item.Item` instead
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
class ItemMeta(_BaseItemMeta):
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
.. _metaclass: https://realpython.com/python-metaclasses
|
||||
|
|
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
|
|||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
fields: Dict[str, Field] = {}
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
||||
fields: Dict[str, Field]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __iter__(self):
|
||||
return iter(self._values)
|
||||
|
||||
__hash__ = BaseItem.__hash__
|
||||
__hash__ = object_ref.__hash__
|
||||
|
||||
def keys(self):
|
||||
return self._values.keys()
|
||||
|
|
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
|
|||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
|
|||
return len(self.attributes)
|
||||
|
||||
def _to_dict(self):
|
||||
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
for k, v in self.items()}
|
||||
|
||||
def _get_key(self, key_value):
|
||||
return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
|
||||
else str(key_value))
|
||||
|
||||
def copy_to_dict(self):
|
||||
"""
|
||||
Make a copy of current settings and convert to a dict.
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
|
|||
process_results methods for pre and post-processing purposes.
|
||||
"""
|
||||
|
||||
for row in csviter(response, self.delimiter, self.headers, self.quotechar):
|
||||
for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
|
||||
ret = iterate_spider_output(self.parse_row(response, row))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
|
|||
|
||||
def deferred_to_future(d: Deferred) -> Future:
|
||||
"""
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return an :class:`asyncio.Future` object that wraps *d*.
|
||||
|
||||
|
|
@ -325,7 +325,7 @@ def deferred_to_future(d: Deferred) -> Future:
|
|||
|
||||
def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
|
||||
"""
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.6.0
|
||||
|
||||
Return *d* as an object that can be awaited from a :ref:`Scrapy callable
|
||||
defined as a coroutine <coroutine-support>`.
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ from w3lib.html import replace_entities
|
|||
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
|
||||
|
||||
|
||||
def arg_to_iter(arg):
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from scrapy.http.response import Response
|
|||
|
||||
from twisted.web import http
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from w3lib import html
|
||||
|
||||
|
||||
|
|
@ -51,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
|
|||
return f'{status_int} {to_unicode(message)}'
|
||||
|
||||
|
||||
@deprecated
|
||||
def response_httprepr(response: Response) -> bytes:
|
||||
"""Return raw HTTP representation (as bytes) of the given response. This
|
||||
is provided only for reference, since it's not the exact stream of bytes
|
||||
|
|
|
|||
1
setup.py
1
setup.py
|
|
@ -32,6 +32,7 @@ install_requires = [
|
|||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
'setuptools',
|
||||
'tldextract',
|
||||
]
|
||||
extras_require = {}
|
||||
cpython_dependencies = [
|
||||
|
|
|
|||
|
|
@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl
|
|||
from twisted.internet.task import deferLater
|
||||
from twisted.names import dns, error
|
||||
from twisted.names.server import DNSServerFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper, Resource
|
||||
from twisted.web import resource, server
|
||||
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
|
||||
from twisted.web.static import File
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.util import redirectTo
|
||||
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
|
@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None):
|
|||
return default
|
||||
|
||||
|
||||
class LeafResource(Resource):
|
||||
# most of the following resources are copied from twisted.web.test.test_webclient
|
||||
class ForeverTakingResource(resource.Resource):
|
||||
"""
|
||||
L{ForeverTakingResource} is a resource which never finishes responding
|
||||
to requests.
|
||||
"""
|
||||
|
||||
def __init__(self, write=False):
|
||||
resource.Resource.__init__(self)
|
||||
self._write = write
|
||||
|
||||
def render(self, request):
|
||||
if self._write:
|
||||
request.write(b"some bytes")
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
||||
class ErrorResource(resource.Resource):
|
||||
def render(self, request):
|
||||
request.setResponseCode(401)
|
||||
if request.args.get(b"showlength"):
|
||||
request.setHeader(b"content-length", b"0")
|
||||
return b""
|
||||
|
||||
|
||||
class NoLengthResource(resource.Resource):
|
||||
def render(self, request):
|
||||
return b"nolength"
|
||||
|
||||
|
||||
class HostHeaderResource(resource.Resource):
|
||||
"""
|
||||
A testing resource which renders itself as the value of the host header
|
||||
from the request.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
return request.requestHeaders.getRawHeaders(b"host")[0]
|
||||
|
||||
|
||||
class PayloadResource(resource.Resource):
|
||||
"""
|
||||
A testing resource which renders itself as the contents of the request body
|
||||
as long as the request body is 100 bytes long, otherwise which renders
|
||||
itself as C{"ERROR"}.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
data = request.content.read()
|
||||
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
|
||||
if len(data) != 100 or int(contentLength) != 100:
|
||||
return b"ERROR"
|
||||
return data
|
||||
|
||||
|
||||
class BrokenDownloadResource(resource.Resource):
|
||||
def render(self, request):
|
||||
# only sends 3 bytes even though it claims to send 5
|
||||
request.setHeader(b"content-length", b"5")
|
||||
request.write(b"abc")
|
||||
return b""
|
||||
|
||||
|
||||
class LeafResource(resource.Resource):
|
||||
|
||||
isLeaf = True
|
||||
|
||||
|
|
@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource):
|
|||
return request.content.read()
|
||||
|
||||
|
||||
class Root(Resource):
|
||||
class Root(resource.Resource):
|
||||
|
||||
def __init__(self):
|
||||
Resource.__init__(self)
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"status", Status())
|
||||
self.putChild(b"follow", Follow())
|
||||
self.putChild(b"delay", Delay())
|
||||
|
|
@ -187,7 +249,7 @@ class Root(Resource):
|
|||
self.putChild(b"raw", Raw())
|
||||
self.putChild(b"echo", Echo())
|
||||
self.putChild(b"payload", PayloadResource())
|
||||
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
|
||||
try:
|
||||
from tests import tests_datadir
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ attrs
|
|||
dataclasses; python_version == '3.6'
|
||||
pyftpdlib
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-cov==3.0.0
|
||||
pytest-xdist
|
||||
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
||||
testfixtures
|
||||
|
|
|
|||
|
|
@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase):
|
|||
settingsdict = json.loads(settingsstr)
|
||||
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
|
||||
self.assertEqual(200, settingsdict[EXT_PATH])
|
||||
|
||||
def test_pathlib_path_as_feeds_key(self):
|
||||
self.assertEqual(self._execute('settings', '--get', 'FEEDS'),
|
||||
json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
from pathlib import Path
|
||||
|
||||
EXTENSIONS = {
|
||||
'tests.test_cmdline.extensions.TestExtension': 0,
|
||||
}
|
||||
|
||||
TEST1 = 'default'
|
||||
|
||||
FEEDS = {
|
||||
Path('items.csv'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ import scrapy
|
|||
|
||||
class CheckSpider(scrapy.Spider):
|
||||
name = '{self.spider_name}'
|
||||
start_urls = ['http://example.com']
|
||||
start_urls = ['http://toscrape.com']
|
||||
|
||||
def parse(self, response, **cb_kwargs):
|
||||
\"\"\"
|
||||
@url http://example.com
|
||||
@url http://toscrape.com
|
||||
{contracts}
|
||||
\"\"\"
|
||||
{parse_def}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import os
|
||||
import argparse
|
||||
from os.path import join, abspath, isfile, exists
|
||||
from twisted.internet import defer
|
||||
from scrapy.commands import parse
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.testsite import SiteTest
|
||||
from scrapy.utils.testproc import ProcessTest
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
|
|||
content = '[\n{},\n{"foo": "bar"}\n]'
|
||||
with open(file_path, 'r') as f:
|
||||
self.assertEqual(f.read(), content)
|
||||
|
||||
def test_parse_add_options(self):
|
||||
command = parse.Command()
|
||||
command.settings = Settings()
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='scrapy', formatter_class=argparse.HelpFormatter,
|
||||
conflict_handler='resolve', prefix_chars='-'
|
||||
)
|
||||
command.add_options(parser)
|
||||
namespace = parser.parse_args(
|
||||
['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
|
||||
)
|
||||
self.assertTrue(namespace.nolinks)
|
||||
self.assertEqual(namespace.depth, 2)
|
||||
self.assertEqual(namespace.spider, self.spider_name)
|
||||
self.assertTrue(namespace.verbose)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import inspect
|
||||
import json
|
||||
import optparse
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
|
|
@ -23,7 +23,7 @@ from twisted.python.versions import Version
|
|||
from twisted.trial import unittest
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.commands.startproject import IGNORE
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
|
|||
def setUp(self):
|
||||
self.command = ScrapyCommand()
|
||||
self.command.settings = Settings()
|
||||
self.parser = optparse.OptionParser(
|
||||
formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve',
|
||||
)
|
||||
self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
conflict_handler='resolve')
|
||||
self.command.add_options(self.parser)
|
||||
|
||||
def test_settings_json_string(self):
|
||||
feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
|
||||
opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
|
||||
opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
|
||||
self.command.process_options(args, opts)
|
||||
self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
|
||||
self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
|
||||
|
||||
def test_help_formatter(self):
|
||||
formatter = ScrapyHelpFormatter(prog='scrapy')
|
||||
part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
|
||||
'\n', 'optional arguments:\n', '\n', 'Global Options:\n']
|
||||
self.assertEqual(
|
||||
formatter._join_parts(part_strings),
|
||||
('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'
|
||||
'Optional Arguments\n==================\n\n'
|
||||
'Global Options\n--------------\n')
|
||||
)
|
||||
|
||||
|
||||
class ProjectTest(unittest.TestCase):
|
||||
project_name = 'testproject'
|
||||
|
|
@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):
|
|||
self.assertNotIn('Unhandled Error', log)
|
||||
|
||||
|
||||
class ViewCommandTest(CommandTest):
|
||||
|
||||
def test_methods(self):
|
||||
command = view.Command()
|
||||
command.settings = Settings()
|
||||
parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
|
||||
formatter_class=ScrapyHelpFormatter,
|
||||
conflict_handler='resolve')
|
||||
command.add_options(parser)
|
||||
self.assertEqual(command.short_desc(),
|
||||
"Open URL in browser, as seen by Scrapy")
|
||||
self.assertIn("URL using the Scrapy downloader and show its",
|
||||
command.long_desc())
|
||||
|
||||
|
||||
class CrawlCommandTest(CommandTest):
|
||||
|
||||
def crawl(self, code, args=()):
|
||||
|
|
|
|||
|
|
@ -15,8 +15,6 @@ from twisted.trial import unittest
|
|||
from twisted.web import resource, server, static, util
|
||||
from twisted.web._newclient import ResponseFailed
|
||||
from twisted.web.http import _DataLoss
|
||||
from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource,
|
||||
NoLengthResource, PayloadResource)
|
||||
from w3lib.url import path_to_file_uri
|
||||
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
|
@ -34,7 +32,15 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.test import get_crawler, skip_if_no_boto
|
||||
from tests.mockserver import MockServer, ssl_context_factory, Echo
|
||||
from tests.mockserver import (
|
||||
Echo,
|
||||
ForeverTakingResource,
|
||||
HostHeaderResource,
|
||||
MockServer,
|
||||
NoLengthResource,
|
||||
PayloadResource,
|
||||
ssl_context_factory,
|
||||
)
|
||||
from tests.spiders import SingleRequestSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,57 @@ import pytest
|
|||
|
||||
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
||||
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
def _cookie_to_set_cookie_value(cookie):
|
||||
"""Given a cookie defined as a dictionary with name and value keys, and
|
||||
optional path and domain keys, return the equivalent string that can be
|
||||
associated to a ``Set-Cookie`` header."""
|
||||
decoded = {}
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
if cookie.get(key) is None:
|
||||
if key in ("name", "value"):
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], (bool, float, int, str)):
|
||||
decoded[key] = str(cookie[key])
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
decoded[key] = cookie[key].decode("latin1", errors="replace")
|
||||
|
||||
cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
|
||||
for key, value in decoded.items(): # path, domain
|
||||
cookie_str += f"; {key.capitalize()}={value}"
|
||||
return cookie_str
|
||||
|
||||
|
||||
def _cookies_to_set_cookie_list(cookies):
|
||||
"""Given a group of cookie defined either as a dictionary or as a list of
|
||||
dictionaries (i.e. in a format supported by the cookies parameter of
|
||||
Request), return the equivalen list of strings that can be associated to a
|
||||
``Set-Cookie`` header."""
|
||||
if not cookies:
|
||||
return []
|
||||
if isinstance(cookies, dict):
|
||||
cookies = ({"name": k, "value": v} for k, v in cookies.items())
|
||||
return filter(
|
||||
None,
|
||||
(
|
||||
_cookie_to_set_cookie_value(cookie)
|
||||
for cookie in cookies
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class CookiesMiddlewareTest(TestCase):
|
||||
|
||||
def assertCookieValEqual(self, first, second, msg=None):
|
||||
|
|
@ -23,9 +67,11 @@ class CookiesMiddlewareTest(TestCase):
|
|||
def setUp(self):
|
||||
self.spider = Spider('foo')
|
||||
self.mw = CookiesMiddleware()
|
||||
self.redirect_middleware = RedirectMiddleware(settings=Settings())
|
||||
|
||||
def tearDown(self):
|
||||
del self.mw
|
||||
del self.redirect_middleware
|
||||
|
||||
def test_basic(self):
|
||||
req = Request('http://scrapytest.org/')
|
||||
|
|
@ -368,3 +414,282 @@ class CookiesMiddlewareTest(TestCase):
|
|||
req4 = Request('http://example.org', cookies={'a': 'b'})
|
||||
assert self.mw.process_request(req4, self.spider) is None
|
||||
self.assertCookieValEqual(req4.headers['Cookie'], b'a=b')
|
||||
|
||||
def _test_cookie_redirect(
|
||||
self,
|
||||
source,
|
||||
target,
|
||||
*,
|
||||
cookies1,
|
||||
cookies2,
|
||||
):
|
||||
input_cookies = {'a': 'b'}
|
||||
|
||||
if not isinstance(source, dict):
|
||||
source = {'url': source}
|
||||
if not isinstance(target, dict):
|
||||
target = {'url': target}
|
||||
target.setdefault('status', 301)
|
||||
|
||||
request1 = Request(cookies=input_cookies, **source)
|
||||
self.mw.process_request(request1, self.spider)
|
||||
cookies = request1.headers.get('Cookie')
|
||||
self.assertEqual(cookies, b"a=b" if cookies1 else None)
|
||||
|
||||
response = Response(
|
||||
headers={
|
||||
'Location': target['url'],
|
||||
},
|
||||
**target,
|
||||
)
|
||||
self.assertEqual(
|
||||
self.mw.process_response(request1, response, self.spider),
|
||||
response,
|
||||
)
|
||||
|
||||
request2 = self.redirect_middleware.process_response(
|
||||
request1,
|
||||
response,
|
||||
self.spider,
|
||||
)
|
||||
self.assertIsInstance(request2, Request)
|
||||
|
||||
self.mw.process_request(request2, self.spider)
|
||||
cookies = request2.headers.get('Cookie')
|
||||
self.assertEqual(cookies, b"a=b" if cookies2 else None)
|
||||
|
||||
def test_cookie_redirect_same_domain(self):
|
||||
self._test_cookie_redirect(
|
||||
'https://toscrape.com',
|
||||
'https://toscrape.com',
|
||||
cookies1=True,
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def test_cookie_redirect_same_domain_forcing_get(self):
|
||||
self._test_cookie_redirect(
|
||||
'https://toscrape.com',
|
||||
{'url': 'https://toscrape.com', 'status': 302},
|
||||
cookies1=True,
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def test_cookie_redirect_different_domain(self):
|
||||
self._test_cookie_redirect(
|
||||
'https://toscrape.com',
|
||||
'https://example.com',
|
||||
cookies1=True,
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def test_cookie_redirect_different_domain_forcing_get(self):
|
||||
self._test_cookie_redirect(
|
||||
'https://toscrape.com',
|
||||
{'url': 'https://example.com', 'status': 302},
|
||||
cookies1=True,
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def _test_cookie_header_redirect(
|
||||
self,
|
||||
source,
|
||||
target,
|
||||
*,
|
||||
cookies2,
|
||||
):
|
||||
"""Test the handling of a user-defined Cookie header when building a
|
||||
redirect follow-up request.
|
||||
|
||||
We follow RFC 6265 for cookie handling. The Cookie header can only
|
||||
contain a list of key-value pairs (i.e. no additional cookie
|
||||
parameters like Domain or Path). Because of that, we follow the same
|
||||
rules that we would follow for the handling of the Set-Cookie response
|
||||
header when the Domain is not set: the cookies must be limited to the
|
||||
target URL domain (not even subdomains can receive those cookies).
|
||||
|
||||
.. note:: This method tests the scenario where the cookie middleware is
|
||||
disabled. Because of known issue #1992, when the cookies
|
||||
middleware is enabled we do not need to be concerned about
|
||||
the Cookie header getting leaked to unintended domains,
|
||||
because the middleware empties the header from every request.
|
||||
"""
|
||||
if not isinstance(source, dict):
|
||||
source = {'url': source}
|
||||
if not isinstance(target, dict):
|
||||
target = {'url': target}
|
||||
target.setdefault('status', 301)
|
||||
|
||||
request1 = Request(headers={'Cookie': b'a=b'}, **source)
|
||||
|
||||
response = Response(
|
||||
headers={
|
||||
'Location': target['url'],
|
||||
},
|
||||
**target,
|
||||
)
|
||||
|
||||
request2 = self.redirect_middleware.process_response(
|
||||
request1,
|
||||
response,
|
||||
self.spider,
|
||||
)
|
||||
self.assertIsInstance(request2, Request)
|
||||
|
||||
cookies = request2.headers.get('Cookie')
|
||||
self.assertEqual(cookies, b"a=b" if cookies2 else None)
|
||||
|
||||
def test_cookie_header_redirect_same_domain(self):
|
||||
self._test_cookie_header_redirect(
|
||||
'https://toscrape.com',
|
||||
'https://toscrape.com',
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def test_cookie_header_redirect_same_domain_forcing_get(self):
|
||||
self._test_cookie_header_redirect(
|
||||
'https://toscrape.com',
|
||||
{'url': 'https://toscrape.com', 'status': 302},
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def test_cookie_header_redirect_different_domain(self):
|
||||
self._test_cookie_header_redirect(
|
||||
'https://toscrape.com',
|
||||
'https://example.com',
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def test_cookie_header_redirect_different_domain_forcing_get(self):
|
||||
self._test_cookie_header_redirect(
|
||||
'https://toscrape.com',
|
||||
{'url': 'https://example.com', 'status': 302},
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def _test_user_set_cookie_domain_followup(
|
||||
self,
|
||||
url1,
|
||||
url2,
|
||||
domain,
|
||||
*,
|
||||
cookies1,
|
||||
cookies2,
|
||||
):
|
||||
input_cookies = [
|
||||
{
|
||||
'name': 'a',
|
||||
'value': 'b',
|
||||
'domain': domain,
|
||||
}
|
||||
]
|
||||
|
||||
request1 = Request(url1, cookies=input_cookies)
|
||||
self.mw.process_request(request1, self.spider)
|
||||
cookies = request1.headers.get('Cookie')
|
||||
self.assertEqual(cookies, b"a=b" if cookies1 else None)
|
||||
|
||||
request2 = Request(url2)
|
||||
self.mw.process_request(request2, self.spider)
|
||||
cookies = request2.headers.get('Cookie')
|
||||
self.assertEqual(cookies, b"a=b" if cookies2 else None)
|
||||
|
||||
def test_user_set_cookie_domain_suffix_private(self):
|
||||
self._test_user_set_cookie_domain_followup(
|
||||
'https://books.toscrape.com',
|
||||
'https://quotes.toscrape.com',
|
||||
'toscrape.com',
|
||||
cookies1=True,
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def test_user_set_cookie_domain_suffix_public_period(self):
|
||||
self._test_user_set_cookie_domain_followup(
|
||||
'https://foo.co.uk',
|
||||
'https://bar.co.uk',
|
||||
'co.uk',
|
||||
cookies1=False,
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def test_user_set_cookie_domain_suffix_public_private(self):
|
||||
self._test_user_set_cookie_domain_followup(
|
||||
'https://foo.blogspot.com',
|
||||
'https://bar.blogspot.com',
|
||||
'blogspot.com',
|
||||
cookies1=False,
|
||||
cookies2=False,
|
||||
)
|
||||
|
||||
def test_user_set_cookie_domain_public_period(self):
|
||||
self._test_user_set_cookie_domain_followup(
|
||||
'https://co.uk',
|
||||
'https://co.uk',
|
||||
'co.uk',
|
||||
cookies1=True,
|
||||
cookies2=True,
|
||||
)
|
||||
|
||||
def _test_server_set_cookie_domain_followup(
|
||||
self,
|
||||
url1,
|
||||
url2,
|
||||
domain,
|
||||
*,
|
||||
cookies,
|
||||
):
|
||||
request1 = Request(url1)
|
||||
self.mw.process_request(request1, self.spider)
|
||||
|
||||
input_cookies = [
|
||||
{
|
||||
'name': 'a',
|
||||
'value': 'b',
|
||||
'domain': domain,
|
||||
}
|
||||
]
|
||||
|
||||
headers = {
|
||||
'Set-Cookie': _cookies_to_set_cookie_list(input_cookies),
|
||||
}
|
||||
response = Response(url1, status=200, headers=headers)
|
||||
self.assertEqual(
|
||||
self.mw.process_response(request1, response, self.spider),
|
||||
response,
|
||||
)
|
||||
|
||||
request2 = Request(url2)
|
||||
self.mw.process_request(request2, self.spider)
|
||||
actual_cookies = request2.headers.get('Cookie')
|
||||
self.assertEqual(actual_cookies, b"a=b" if cookies else None)
|
||||
|
||||
def test_server_set_cookie_domain_suffix_private(self):
|
||||
self._test_server_set_cookie_domain_followup(
|
||||
'https://books.toscrape.com',
|
||||
'https://quotes.toscrape.com',
|
||||
'toscrape.com',
|
||||
cookies=True,
|
||||
)
|
||||
|
||||
def test_server_set_cookie_domain_suffix_public_period(self):
|
||||
self._test_server_set_cookie_domain_followup(
|
||||
'https://foo.co.uk',
|
||||
'https://bar.co.uk',
|
||||
'co.uk',
|
||||
cookies=False,
|
||||
)
|
||||
|
||||
def test_server_set_cookie_domain_suffix_public_private(self):
|
||||
self._test_server_set_cookie_domain_followup(
|
||||
'https://foo.blogspot.com',
|
||||
'https://bar.blogspot.com',
|
||||
'blogspot.com',
|
||||
cookies=False,
|
||||
)
|
||||
|
||||
def test_server_set_cookie_domain_public_period(self):
|
||||
self._test_server_set_cookie_domain_followup(
|
||||
'https://co.uk',
|
||||
'https://co.uk',
|
||||
'co.uk',
|
||||
cookies=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
from itertools import product
|
||||
from unittest import TestCase
|
||||
|
||||
from scrapy.downloadermiddlewares.stats import DownloaderStats
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase):
|
|||
self.mw.process_response(self.req, self.res, self.spider)
|
||||
self.assertStatsEqual('downloader/response_count', 1)
|
||||
|
||||
def test_response_len(self):
|
||||
body = (b'', b'not_empty') # empty/notempty body
|
||||
headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h
|
||||
test_responses = [ # form test responses with all combinations of body/headers
|
||||
Response(
|
||||
url='scrapytest.org',
|
||||
status=200,
|
||||
body=r[0],
|
||||
headers=r[1]
|
||||
)
|
||||
for r in product(body, headers)
|
||||
]
|
||||
for test_response in test_responses:
|
||||
self.crawler.stats.set_value('downloader/response_bytes', 0)
|
||||
self.mw.process_response(self.req, test_response, self.spider)
|
||||
self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
|
||||
|
||||
def test_process_exception(self):
|
||||
self.mw.process_exception(self.req, MyException(), self.spider)
|
||||
self.assertStatsEqual('downloader/exception_count', 1)
|
||||
|
|
|
|||
|
|
@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
|||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class URIParamsTest:
|
||||
|
||||
spider_name = "uri_params_spider"
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def test_default(self):
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_none(self):
|
||||
def uri_params(params, spider):
|
||||
pass
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
'Modifying the params dictionary in-place in the '
|
||||
'function defined in the FEED_URI_PARAMS setting or '
|
||||
'in the uri_params key of the FEEDS setting is '
|
||||
'deprecated. The function must return a new '
|
||||
'dictionary instead.'
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_empty_dict(self):
|
||||
def uri_params(params, spider):
|
||||
return {}
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
with self.assertRaises(KeyError):
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
def test_params_as_is(self):
|
||||
def uri_params(params, spider):
|
||||
return params
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_custom_param(self):
|
||||
def uri_params(params, spider):
|
||||
return {**params, 'foo': self.spider_name}
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(foo)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
|
||||
class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
extra_settings = {}
|
||||
if uri_params:
|
||||
extra_settings['FEED_URI_PARAMS'] = uri_params
|
||||
return {
|
||||
'FEED_URI': uri,
|
||||
**extra_settings,
|
||||
}
|
||||
|
||||
|
||||
class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
options = {
|
||||
'format': 'jl',
|
||||
}
|
||||
if uri_params:
|
||||
options['uri_params'] = uri_params
|
||||
return {
|
||||
'FEEDS': {
|
||||
uri: options,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ class WrappedRequestTest(TestCase):
|
|||
self.assertTrue(self.wrapped.unverifiable)
|
||||
|
||||
def test_get_origin_req_host(self):
|
||||
self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com')
|
||||
self.assertEqual(self.wrapped.origin_req_host, 'www.example.com')
|
||||
|
||||
def test_has_header(self):
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
import unittest
|
||||
from unittest import mock
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from w3lib.encoding import resolve_encoding
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
|
||||
XmlResponse, Headers)
|
||||
from scrapy.selector import Selector
|
||||
|
|
@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase):
|
|||
assert isinstance(response.text, str)
|
||||
self._assert_response_encoding(response, encoding)
|
||||
self.assertEqual(response.body, body_bytes)
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertEqual(response.body_as_unicode(), body_unicode)
|
||||
self.assertEqual(response.text, body_unicode)
|
||||
|
||||
def _assert_response_encoding(self, response, encoding):
|
||||
|
|
@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest):
|
|||
original_string = unicode_string.encode('cp1251')
|
||||
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
|
||||
|
||||
# check body_as_unicode
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertTrue(isinstance(r1.body_as_unicode(), str))
|
||||
self.assertEqual(r1.body_as_unicode(), unicode_string)
|
||||
|
||||
# check response.text
|
||||
self.assertTrue(isinstance(r1.text, str))
|
||||
self.assertEqual(r1.text, unicode_string)
|
||||
|
|
@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest):
|
|||
with self.assertRaises(ValueError):
|
||||
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
|
||||
|
||||
def test_body_as_unicode_deprecation_warning(self):
|
||||
with catch_warnings(record=True) as warnings:
|
||||
r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
|
||||
self.assertEqual(r1.body_as_unicode(), 'Hello')
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
def test_json_response(self):
|
||||
json_body = b"""{"ip": "109.187.217.200"}"""
|
||||
json_response = self.response_class("http://www.example.com", body=json_body)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import unittest
|
||||
from unittest import mock
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
|
||||
from scrapy.item import ABCMeta, Field, Item, ItemMeta
|
||||
|
||||
|
||||
class ItemTest(unittest.TestCase):
|
||||
|
|
@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase):
|
|||
item['tags'].append('tag2')
|
||||
assert item['tags'] != copied_item['tags']
|
||||
|
||||
def test_dictitem_deprecation_warning(self):
|
||||
"""Make sure the DictItem deprecation warning is not issued for
|
||||
Item"""
|
||||
with catch_warnings(record=True) as warnings:
|
||||
Item()
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
SubclassedItem()
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
|
||||
class ItemMetaTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -303,94 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase):
|
|||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class DictItemTest(unittest.TestCase):
|
||||
|
||||
def test_deprecation_warning(self):
|
||||
with catch_warnings(record=True) as warnings:
|
||||
DictItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
with catch_warnings(record=True) as warnings:
|
||||
class SubclassedDictItem(DictItem):
|
||||
pass
|
||||
SubclassedDictItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
class BaseItemTest(unittest.TestCase):
|
||||
|
||||
def test_isinstance_check(self):
|
||||
|
||||
class SubclassedBaseItem(BaseItem):
|
||||
pass
|
||||
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertTrue(isinstance(BaseItem(), BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
|
||||
self.assertTrue(isinstance(Item(), BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedItem(), BaseItem))
|
||||
|
||||
# make sure internal checks using private _BaseItem class succeed
|
||||
self.assertTrue(isinstance(BaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(Item(), _BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
|
||||
|
||||
def test_deprecation_warning(self):
|
||||
"""
|
||||
Make sure deprecation warnings are logged whenever BaseItem is used,
|
||||
either instantiated or in an isinstance check
|
||||
"""
|
||||
with catch_warnings(record=True) as warnings:
|
||||
BaseItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
|
||||
class SubclassedBaseItem(BaseItem):
|
||||
pass
|
||||
|
||||
SubclassedBaseItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
self.assertFalse(isinstance("foo", BaseItem))
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
self.assertTrue(isinstance(BaseItem(), BaseItem))
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
class ItemNoDeprecationWarningTest(unittest.TestCase):
|
||||
def test_no_deprecation_warning(self):
|
||||
"""
|
||||
Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
|
||||
"""
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
Item()
|
||||
SubclassedItem()
|
||||
_BaseItem()
|
||||
self.assertFalse(isinstance("foo", _BaseItem))
|
||||
self.assertFalse(isinstance("foo", Item))
|
||||
self.assertFalse(isinstance("foo", SubclassedItem))
|
||||
self.assertTrue(isinstance(_BaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(Item(), Item))
|
||||
self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from scrapy.spiders import (
|
|||
)
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
class SpiderTest(unittest.TestCase):
|
||||
|
|
@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest):
|
|||
|
||||
spider_class = CSVFeedSpider
|
||||
|
||||
def test_parse_rows(self):
|
||||
body = get_testdata('feeds', 'feed-sample6.csv')
|
||||
response = Response("http://example.org/dummy.csv", body=body)
|
||||
|
||||
class _CrawlSpider(self.spider_class):
|
||||
name = "test"
|
||||
delimiter = ","
|
||||
quotechar = "'"
|
||||
|
||||
def parse_row(self, response, row):
|
||||
return row
|
||||
|
||||
spider = _CrawlSpider()
|
||||
rows = list(spider.parse_rows(response))
|
||||
assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'}
|
||||
assert len(rows) == 4
|
||||
|
||||
|
||||
class CrawlSpiderTest(SpiderTest):
|
||||
|
||||
|
|
|
|||
|
|
@ -21,14 +21,6 @@ except ImportError:
|
|||
from twisted.python.filepath import FilePath
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.web.test.test_webclient import (
|
||||
ForeverTakingResource,
|
||||
ErrorResource,
|
||||
NoLengthResource,
|
||||
HostHeaderResource,
|
||||
PayloadResource,
|
||||
BrokenDownloadResource,
|
||||
)
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
|
||||
|
|
@ -36,7 +28,15 @@ from scrapy.http import Request, Headers
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from tests.mockserver import ssl_context_factory
|
||||
from tests.mockserver import (
|
||||
BrokenDownloadResource,
|
||||
ErrorResource,
|
||||
ForeverTakingResource,
|
||||
HostHeaderResource,
|
||||
NoLengthResource,
|
||||
PayloadResource,
|
||||
ssl_context_factory,
|
||||
)
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
|
||||
|
|
|
|||
9
tox.ini
9
tox.ini
|
|
@ -17,6 +17,8 @@ deps =
|
|||
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
|
||||
# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
|
||||
markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
|
||||
# Extras
|
||||
botocore>=1.4.87
|
||||
passenv =
|
||||
|
|
@ -47,7 +49,7 @@ commands =
|
|||
[testenv:security]
|
||||
basepython = python3
|
||||
deps =
|
||||
bandit
|
||||
bandit==1.7.3
|
||||
commands =
|
||||
bandit -r -c .bandit.yml {posargs:scrapy}
|
||||
|
||||
|
|
@ -66,7 +68,7 @@ commands =
|
|||
basepython = python3
|
||||
deps =
|
||||
{[testenv:extra-deps]deps}
|
||||
pylint==2.12.1
|
||||
pylint==2.12.2
|
||||
commands =
|
||||
pylint conftest.py docs extras scrapy setup.py tests
|
||||
|
||||
|
|
@ -127,6 +129,9 @@ deps =
|
|||
robotexclusionrulesparser
|
||||
Pillow>=4.0.0
|
||||
Twisted[http2]>=17.9.0
|
||||
# Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
|
||||
# so we need to pin old markupsafe here too
|
||||
markupsafe < 2.1.0
|
||||
|
||||
[testenv:asyncio]
|
||||
commands =
|
||||
|
|
|
|||
Loading…
Reference in New Issue