mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into per_slot_settings
This commit is contained in:
GeorgeA92
commit
ac55a933b8
|
|
@ -19,7 +19,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: pylint
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
- python-version: 3.6
|
||||
env:
|
||||
TOXENV: typing
|
||||
|
|
@ -38,8 +37,5 @@ jobs:
|
|||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ jobs:
|
|||
- python-version: 3.8
|
||||
env:
|
||||
TOXENV: extra-deps
|
||||
TOX_PIP_VERSION: 20.3.3
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -73,9 +72,6 @@ jobs:
|
|||
$PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [[ ! -z "$TOX_PIP_VERSION" ]]; then
|
||||
pip install tox-pip-version
|
||||
fi
|
||||
pip install -U tox
|
||||
tox
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ jobs:
|
|||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: asyncio
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
.. image:: /artwork/scrapy-logo.jpg
|
||||
:width: 400px
|
||||
.. image:: https://scrapy.org/img/scrapylogo.png
|
||||
|
||||
======
|
||||
Scrapy
|
||||
|
|
|
|||
|
|
@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
|
|||
r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
|
||||
|
||||
# Never documented before, and deprecated now.
|
||||
r'^scrapy\.item\.DictItem$',
|
||||
r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
|
||||
|
||||
# Implementation detail of LxmlLinkExtractor
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Examples
|
|||
The best way to learn is with examples, and Scrapy is no exception. For this
|
||||
reason, there is an example Scrapy project named quotesbot_, that you can use to
|
||||
play and learn more about Scrapy. It contains two spiders for
|
||||
http://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
https://quotes.toscrape.com, one using CSS selectors and another one using XPath
|
||||
expressions.
|
||||
|
||||
The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
|
|||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
* *(Recommended)* **Don't** use system Python. Install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
|
|
@ -234,8 +234,8 @@ For PyPy3, only Linux installation was tested.
|
|||
|
||||
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependencies will be built during installation.
|
||||
On macOS, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
On macOS, you are likely to face an issue with building the Cryptography
|
||||
dependency. The solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing Scrapy). Installing on Linux has no special
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
|
|||
example of a Scrapy Spider using the simplest way to run a spider.
|
||||
|
||||
Here's the code for a spider that scrapes famous quotes from website
|
||||
http://quotes.toscrape.com, following the pagination::
|
||||
https://quotes.toscrape.com, following the pagination::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/tag/humor/',
|
||||
'https://quotes.toscrape.com/tag/humor/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Scrapy Tutorial
|
|||
In this tutorial, we'll assume that Scrapy is already installed on your system.
|
||||
If that's not the case, see :ref:`intro-install`.
|
||||
|
||||
We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
|
||||
We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
|
||||
that lists quotes from famous authors.
|
||||
|
||||
This tutorial will walk you through these tasks:
|
||||
|
|
@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
|
|||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
|
@ -143,9 +143,9 @@ similar to this::
|
|||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
|
||||
2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
|
||||
2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
|
||||
2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
|
@ -184,8 +184,8 @@ for your spider::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -207,7 +207,7 @@ Extracting data
|
|||
The best way to learn how to extract data with Scrapy is trying selectors
|
||||
using the :ref:`Scrapy shell <topics-shell>`. Run::
|
||||
|
||||
scrapy shell 'http://quotes.toscrape.com/page/1/'
|
||||
scrapy shell 'https://quotes.toscrape.com/page/1/'
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "http://quotes.toscrape.com/page/1/"
|
||||
scrapy shell "https://quotes.toscrape.com/page/1/"
|
||||
|
||||
You will see something like::
|
||||
|
||||
[ ... Scrapy log here ... ]
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
|
||||
2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
|
||||
[s] Available Scrapy objects:
|
||||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90>
|
||||
[s] item {}
|
||||
[s] request <GET http://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 http://quotes.toscrape.com/page/1/>
|
||||
[s] request <GET https://quotes.toscrape.com/page/1/>
|
||||
[s] response <200 https://quotes.toscrape.com/page/1/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7fa91d888c10>
|
||||
[s] spider <DefaultSpider 'default' at 0x7fa91c8af990>
|
||||
[s] Useful shortcuts:
|
||||
|
|
@ -241,7 +241,7 @@ object:
|
|||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
|
||||
|
||||
>>> response.css('title')
|
||||
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
|
||||
|
|
@ -355,7 +355,7 @@ Extracting quotes and authors
|
|||
Now that you know a bit about selection and extraction, let's complete our
|
||||
spider by writing the code to extract the quotes from the web page.
|
||||
|
||||
Each quote in http://quotes.toscrape.com is represented by HTML elements that look
|
||||
Each quote in https://quotes.toscrape.com is represented by HTML elements that look
|
||||
like this:
|
||||
|
||||
.. code-block:: html
|
||||
|
|
@ -379,7 +379,7 @@ like this:
|
|||
Let's open up scrapy shell and play a bit to find out how to extract the data
|
||||
we want::
|
||||
|
||||
$ scrapy shell 'http://quotes.toscrape.com'
|
||||
$ scrapy shell 'https://quotes.toscrape.com'
|
||||
|
||||
We get a list of selectors for the quote HTML elements with:
|
||||
|
||||
|
|
@ -444,8 +444,8 @@ in the callback, as you can see below::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'http://quotes.toscrape.com/page/2/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/2/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -458,9 +458,9 @@ in the callback, as you can see below::
|
|||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
|
||||
2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
|
||||
{'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
|
||||
|
||||
|
||||
|
|
@ -505,7 +505,7 @@ Following links
|
|||
===============
|
||||
|
||||
Let's say, instead of just scraping the stuff from the first two pages
|
||||
from http://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
from https://quotes.toscrape.com, you want quotes from all the pages in the website.
|
||||
|
||||
Now that you know how to extract data from pages, let's see how to follow links
|
||||
from them.
|
||||
|
|
@ -549,7 +549,7 @@ page, extracting data from it::
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use
|
|||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
'https://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -654,7 +654,7 @@ this time for scraping author information::
|
|||
class AuthorSpider(scrapy.Spider):
|
||||
name = 'author'
|
||||
|
||||
start_urls = ['http://quotes.toscrape.com/']
|
||||
start_urls = ['https://quotes.toscrape.com/']
|
||||
|
||||
def parse(self, response):
|
||||
author_page_links = response.css('.author + a')
|
||||
|
|
@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument::
|
|||
name = "quotes"
|
||||
|
||||
def start_requests(self):
|
||||
url = 'http://quotes.toscrape.com/'
|
||||
url = 'https://quotes.toscrape.com/'
|
||||
tag = getattr(self, 'tag', None)
|
||||
if tag is not None:
|
||||
url = url + 'tag/' + tag
|
||||
|
|
@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument::
|
|||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
will only visit URLs from the ``humor`` tag, such as
|
||||
``http://quotes.toscrape.com/tag/humor``.
|
||||
``https://quotes.toscrape.com/tag/humor``.
|
||||
|
||||
You can :ref:`learn more about handling spider arguments here <spiderargs>`.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,25 @@
|
|||
.. note::
|
||||
.. versionchanged:: VERSION
|
||||
|
||||
The Twisted reactor is now installed when
|
||||
:meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
|
||||
:class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
|
||||
:setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
|
||||
honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
|
||||
they are silently ignored when set there and you need to set these settings
|
||||
in some other way.
|
||||
|
||||
|
||||
.. note::
|
||||
.. versionchanged:: VERSION
|
||||
|
||||
Previously this setting had no effect in a spider
|
||||
:attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
|
||||
if you :ref:`run several spiders in one process <run-multiple-spiders>`,
|
||||
they must not have different values for this setting, because they will use
|
||||
a single reactor instance.
|
||||
|
||||
|
||||
.. _news:
|
||||
|
||||
Release notes
|
||||
|
|
@ -4358,7 +4380,7 @@ Code rearranged and removed
|
|||
- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ this:
|
|||
the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
|
||||
to crawl.
|
||||
|
||||
9. The process repeats (from step 1) until there are no more requests from the
|
||||
9. The process repeats (from step 3) until there are no more requests from the
|
||||
:ref:`Scheduler <component-scheduler>`.
|
||||
|
||||
Components
|
||||
|
|
|
|||
|
|
@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
|
|||
asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
|
||||
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
|
||||
|
||||
.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
|
||||
recommended for production environments. Future Scrapy versions
|
||||
may introduce related changes without a deprecation period or
|
||||
warning.
|
||||
|
||||
.. _install-asyncio:
|
||||
|
||||
Installing the asyncio reactor
|
||||
|
|
@ -41,6 +36,34 @@ use it instead of the default asyncio event loop.
|
|||
|
||||
.. _asyncio-await-dfd:
|
||||
|
||||
Windows-specific notes
|
||||
======================
|
||||
|
||||
The Windows implementation of :mod:`asyncio` can use two event loop
|
||||
implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
|
||||
3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
|
||||
(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
|
||||
event loop class needs to be changed. Scrapy since VERSION does this
|
||||
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||
:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
|
||||
by other means or use an older Scrapy version you need to call the following
|
||||
code before installing the reactor::
|
||||
|
||||
import asyncio
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
|
||||
You can put this in the same function that installs the reactor, if you do that
|
||||
yourself, or in some code that runs before the reactor is installed, e.g.
|
||||
``settings.py``.
|
||||
|
||||
.. note:: Other libraries you use may require
|
||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||
subprocesses (this is the case with `playwright`_), so you cannot use
|
||||
them together with Scrapy on Windows (but you should be able to use
|
||||
them on WSL or native Linux).
|
||||
|
||||
.. _playwright: https://github.com/microsoft/playwright-python
|
||||
|
||||
Awaiting on Deferreds
|
||||
=====================
|
||||
|
||||
|
|
|
|||
|
|
@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
|
|||
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
|
||||
copy XPaths to selected elements. Let's try it out.
|
||||
|
||||
First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
|
||||
First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
$ scrapy shell "http://quotes.toscrape.com/"
|
||||
$ scrapy shell "https://quotes.toscrape.com/"
|
||||
|
||||
Then, back to your web browser, right-click on the ``span`` tag, select
|
||||
``Copy > XPath`` and paste it in the Scrapy shell like so:
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
response = load_response('http://quotes.toscrape.com/', 'quotes.html')
|
||||
response = load_response('https://quotes.toscrape.com/', 'quotes.html')
|
||||
|
||||
>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
|
||||
['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
|
||||
|
|
@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
|
|||
type ``json``.
|
||||
|
||||
If we click on this request, we see that the request URL is
|
||||
``http://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
``https://quotes.toscrape.com/api/quotes?page=1`` and the response
|
||||
is a JSON-object that contains our quotes. We can also right-click
|
||||
on the request and open ``Open in new tab`` to get a better overview.
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ also request each page to get every quote on the site::
|
|||
name = 'quote'
|
||||
allowed_domains = ['quotes.toscrape.com']
|
||||
page = 1
|
||||
start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
|
||||
start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
|
||||
|
||||
def parse(self, response):
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -255,7 +255,7 @@ also request each page to get every quote on the site::
|
|||
yield {"quote": quote["text"]}
|
||||
if data["has_next"]:
|
||||
self.page += 1
|
||||
url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
This spider starts at the first page of the quotes-API. With each
|
||||
|
|
@ -280,7 +280,7 @@ request::
|
|||
from scrapy import Request
|
||||
|
||||
request = Request.from_curl(
|
||||
"curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
|
||||
"la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
|
||||
"pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
|
||||
"-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
|
||||
|
|
@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
|
|||
to identifying the correct request and replicating it in your spider.
|
||||
|
||||
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
|
||||
.. _quotes.toscrape.com: http://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _quotes.toscrape.com: https://quotes.toscrape.com
|
||||
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
|
||||
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
|
||||
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
|
||||
|
||||
|
|
|
|||
|
|
@ -322,7 +322,7 @@ Post-Processing
|
|||
|
||||
Scrapy provides an option to activate plugins to post-process feeds before they are exported
|
||||
to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
can create your own :ref:`plugins <custom-plugins>`.
|
||||
|
||||
These plugins can be activated through the ``postprocessing`` option of a feed.
|
||||
The option must be passed a list of post-processing plugins in the order you want
|
||||
|
|
@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods:
|
|||
|
||||
Close the target file object.
|
||||
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
|
||||
can then access those parameters from the ``__init__`` method of your plugin.
|
||||
|
||||
|
||||
|
|
@ -744,6 +744,9 @@ The function signature should be as follows:
|
|||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.Spider
|
||||
|
||||
.. caution:: The function should return a new dictionary, modifying
|
||||
the received ``params`` in-place is deprecated.
|
||||
|
||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
|
|
|
|||
|
|
@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many
|
|||
HTTP 404 and 500 responses, and you want to hide all messages like this::
|
||||
|
||||
2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
|
||||
response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
|
||||
is not handled or not allowed
|
||||
|
||||
The first thing to note is a logger name - it is in brackets:
|
||||
|
|
|
|||
|
|
@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
|
|||
crawl()
|
||||
reactor.run() # the script will block here until the last crawl call is finished
|
||||
|
||||
Different spiders can set different values for the same setting, but when they
|
||||
run in the same process it may be impossible, by design or because of some
|
||||
limitations, to use these different values. What happens in practice is
|
||||
different for different settings:
|
||||
|
||||
* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
|
||||
(:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
|
||||
default one) cannot be read from the per-spider settings. These are applied
|
||||
when the :class:`~scrapy.crawler.CrawlerRunner` or
|
||||
:class:`~scrapy.crawler.CrawlerProcess` object is created.
|
||||
* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
|
||||
available value is used, and if a spider requests a different reactor an
|
||||
exception will be raised. These are applied when the reactor is installed.
|
||||
* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
|
||||
ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
|
||||
:setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
|
||||
the first available value is used. These are applied when the reactor is
|
||||
started.
|
||||
|
||||
.. seealso:: :ref:`run-from-script`.
|
||||
|
||||
.. _distributed-crawls:
|
||||
|
|
|
|||
|
|
@ -950,6 +950,14 @@ TextResponse objects
|
|||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
.. method:: TextResponse.urljoin(url)
|
||||
|
||||
Constructs an absolute url by combining the Response's base url with
|
||||
a possible relative url. The base url shall be extracted from the
|
||||
``<base>`` tag, or just the Response's :attr:`url` if there is no such
|
||||
tag.
|
||||
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
--------------------
|
||||
|
|
|
|||
|
|
@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy::
|
|||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes::
|
|||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
urls = ['https://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
|
|
@ -1638,10 +1638,9 @@ which raises :exc:`Exception`, becomes::
|
|||
|
||||
|
||||
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
|
||||
means that Scrapy will not attempt to install any specific reactor, and the
|
||||
default reactor defined by Twisted for the current platform will be used. This
|
||||
is to maintain backward compatibility and avoid possible problems caused by
|
||||
using a non-default reactor.
|
||||
means that Scrapy will install the default reactor defined by Twisted for the
|
||||
current platform. This is to maintain backward compatibility and avoid possible
|
||||
problems caused by using a non-default reactor.
|
||||
|
||||
For additional information, see :doc:`core/howto/choosing-reactor`.
|
||||
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ Let's take an example::
|
|||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
start_urls = ['https://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
|
|
|
|||
1
pylintrc
1
pylintrc
|
|
@ -112,6 +112,7 @@ disable=abstract-method,
|
|||
unused-private-member,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
use-implicit-booleaness-not-comparison,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
|
|
|
|||
|
|
@ -21,5 +21,3 @@ addopts =
|
|||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
|
||||
filterwarnings=
|
||||
ignore::DeprecationWarning:twisted.web.test.test_webclient
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import argparse
|
||||
import cProfile
|
||||
import inspect
|
||||
import pkg_resources
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
|
|
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
|
|||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
|
||||
parser.description = cmd.long_desc()
|
||||
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler='resolve',
|
||||
description=cmd.long_desc())
|
||||
settings.setdict(cmd.default_settings, priority='command')
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
cmd.crawler_process = CrawlerProcess(settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Base class for Scrapy commands
|
||||
"""
|
||||
import os
|
||||
from optparse import OptionGroup
|
||||
import argparse
|
||||
from typing import Any, Dict
|
||||
|
||||
from twisted.python import failure
|
||||
|
|
@ -43,14 +43,14 @@ class ScrapyCommand:
|
|||
|
||||
def long_desc(self):
|
||||
"""A long description of the command. Return short description when not
|
||||
available. It cannot contain newlines, since contents will be formatted
|
||||
available. It cannot contain newlines since contents will be formatted
|
||||
by optparser which removes newlines and wraps text.
|
||||
"""
|
||||
return self.short_desc()
|
||||
|
||||
def help(self):
|
||||
"""An extensive help for the command. It will be shown when using the
|
||||
"help" command. It can contain newlines, since no post-formatting will
|
||||
"help" command. It can contain newlines since no post-formatting will
|
||||
be applied to its contents.
|
||||
"""
|
||||
return self.long_desc()
|
||||
|
|
@ -59,22 +59,20 @@ class ScrapyCommand:
|
|||
"""
|
||||
Populate option parse with options available for this command
|
||||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
group = parser.add_argument_group(title='Global Options')
|
||||
group.add_argument("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help=f"log level (default: {self.settings['LOG_LEVEL']})")
|
||||
group.add_argument("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
group.add_argument("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_argument("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
try:
|
||||
|
|
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_argument("-o", "--output", metavar="FILE", action="append",
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_argument("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
|
||||
class ScrapyHelpFormatter(argparse.HelpFormatter):
|
||||
"""
|
||||
Help Formatter for scrapy command line help messages.
|
||||
"""
|
||||
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
|
||||
super().__init__(prog, indent_increment=indent_increment,
|
||||
max_help_position=max_help_position, width=width)
|
||||
|
||||
def _join_parts(self, part_strings):
|
||||
parts = self.format_part_strings(part_strings)
|
||||
return super()._join_parts(parts)
|
||||
|
||||
def format_part_strings(self, part_strings):
|
||||
"""
|
||||
Underline and title case command line help message headers.
|
||||
"""
|
||||
if part_strings and part_strings[0].startswith("usage: "):
|
||||
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
|
||||
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
|
||||
for index in headings[::-1]:
|
||||
char = '-' if "Global Options" in part_strings[index] else '='
|
||||
part_strings[index] = part_strings[index][:-2].title()
|
||||
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
|
||||
part_strings.insert(index + 1, underline)
|
||||
return part_strings
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="only list contracts, without checking them")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
|
||||
help="print contract tests for all spiders")
|
||||
|
||||
def run(self, args, opts):
|
||||
# load contracts
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
parser.add_argument("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -44,16 +44,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
parser.add_argument("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
parser.add_argument("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
|
|||
|
|
@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
parser.add_argument("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_argument("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_argument("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_argument("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_argument("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
|
|||
|
|
@ -18,16 +18,16 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
parser.add_argument("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_argument("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -33,12 +33,12 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_argument("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_argument("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
@ -75,6 +75,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
def _start_crawler_thread(self):
|
||||
t = Thread(target=self.crawler_process.start,
|
||||
kwargs={'stop_after_crawl': False})
|
||||
kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ class Command(fetch.Command):
|
|||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
parser.add_argument('--headers', help=argparse.SUPPRESS)
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Downloader Middleware manager
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
from typing import Callable, Union
|
||||
from typing import Callable, Union, cast
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
@defer.inlineCallbacks
|
||||
def process_request(request: Request):
|
||||
for method in self.methods['process_request']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
def process_exception(failure: Failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
method = cast(Callable, method)
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from time import time
|
|||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
|
||||
from scrapy.http import Headers
|
||||
|
|
@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
from twisted.internet import reactor
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Spider Middleware manager
|
|||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
from itertools import islice
|
||||
from typing import Any, Callable, Generator, Iterable, Union
|
||||
from typing import Any, Callable, Generator, Iterable, Union, cast
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
|
||||
spider: Spider) -> Any:
|
||||
for method in self.methods['process_spider_input']:
|
||||
method = cast(Callable, method)
|
||||
try:
|
||||
result = method(response=response, spider=spider)
|
||||
if result is not None:
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.utils.log import (
|
|||
configure_logging,
|
||||
get_scrapy_root_handler,
|
||||
install_scrapy_root_handler,
|
||||
log_reactor_info,
|
||||
log_scrapy_info,
|
||||
LogCounterHandler,
|
||||
)
|
||||
|
|
@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class Crawler:
|
||||
|
||||
def __init__(self, spidercls, settings=None):
|
||||
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
|
||||
if isinstance(spidercls, Spider):
|
||||
raise ValueError('The spidercls argument must be a class, not an object')
|
||||
|
||||
|
|
@ -69,6 +70,20 @@ class Crawler:
|
|||
|
||||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
# but before something imports twisted.internet.reactor
|
||||
if reactor_class:
|
||||
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
else:
|
||||
from twisted.internet import default
|
||||
default.install()
|
||||
log_reactor_info()
|
||||
if reactor_class:
|
||||
verify_installed_reactor(reactor_class)
|
||||
|
||||
self.extensions = ExtensionManager.from_crawler(self)
|
||||
|
||||
self.settings.freeze()
|
||||
|
|
@ -153,7 +168,6 @@ class CrawlerRunner:
|
|||
self._crawlers = set()
|
||||
self._active = set()
|
||||
self.bootstrap_failed = False
|
||||
self._handle_twisted_reactor()
|
||||
|
||||
@property
|
||||
def spiders(self):
|
||||
|
|
@ -247,10 +261,6 @@ class CrawlerRunner:
|
|||
while self._active:
|
||||
yield defer.DeferredList(self._active)
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
|
||||
|
||||
|
||||
class CrawlerProcess(CrawlerRunner):
|
||||
"""
|
||||
|
|
@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
||||
|
|
@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner):
|
|||
{'signame': signame})
|
||||
reactor.callFromThread(self._stop_reactor)
|
||||
|
||||
def start(self, stop_after_crawl=True):
|
||||
def _create_crawler(self, spidercls):
|
||||
if isinstance(spidercls, str):
|
||||
spidercls = self.spider_loader.load(spidercls)
|
||||
return Crawler(spidercls, self.settings, init_reactor=True)
|
||||
|
||||
def start(self, stop_after_crawl=True, install_signal_handlers=True):
|
||||
"""
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
|
|
@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
|
||||
:param bool install_signal_handlers: whether to install the shutdown
|
||||
handlers (default: True)
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
if stop_after_crawl:
|
||||
|
|
@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner):
|
|||
return
|
||||
d.addBoth(self._stop_reactor)
|
||||
|
||||
if install_signal_handlers:
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
resolver_class = load_object(self.settings["DNS_RESOLVER"])
|
||||
resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
|
||||
resolver.install_on_reactor()
|
||||
|
|
@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner):
|
|||
reactor.stop()
|
||||
except RuntimeError: # raised if already stopped or in shutdown stage
|
||||
pass
|
||||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,22 @@
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.python import global_object_name, to_bytes
|
||||
from scrapy.utils.request import request_httprepr
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
from twisted.web import http
|
||||
|
||||
|
||||
def get_header_size(headers):
|
||||
size = 0
|
||||
for key, value in headers.items():
|
||||
if isinstance(value, (list, tuple)):
|
||||
for v in value:
|
||||
size += len(b": ") + len(key) + len(v)
|
||||
return size + len(b'\r\n') * (len(headers.keys()) - 1)
|
||||
|
||||
|
||||
def get_status_size(response_status):
|
||||
return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
|
||||
# resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
|
||||
|
||||
|
||||
class DownloaderStats:
|
||||
|
|
@ -24,7 +39,8 @@ class DownloaderStats:
|
|||
def process_response(self, request, response, spider):
|
||||
self.stats.inc_value('downloader/response_count', spider=spider)
|
||||
self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
|
||||
reslen = len(response_httprepr(response))
|
||||
reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
|
||||
# response.body + b"\r\n"+ response.header + b"\r\n" + response.status
|
||||
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
|
||||
|
|
@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
return serializer(value)
|
||||
|
||||
def _serialize_value(self, value):
|
||||
if isinstance(value, _BaseItem):
|
||||
if isinstance(value, Item):
|
||||
return self.export_item(value)
|
||||
elif is_item(value):
|
||||
return dict(self._serialize_item(value))
|
||||
|
|
|
|||
|
|
@ -11,14 +11,14 @@ import sys
|
|||
import warnings
|
||||
from datetime import datetime
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import implementer, Interface
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||
from scrapy.utils.boto import is_botocore_available
|
||||
|
|
@ -524,7 +524,12 @@ class FeedExporter:
|
|||
raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
|
||||
return instance
|
||||
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
def _get_uri_params(
|
||||
self,
|
||||
spider: Spider,
|
||||
uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
|
||||
slot: Optional[_FeedSlot] = None,
|
||||
) -> dict:
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
|
|
@ -532,9 +537,18 @@ class FeedExporter:
|
|||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
original_params = params.copy()
|
||||
uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
|
||||
new_params = uripar_function(params, spider)
|
||||
if new_params is None or original_params != params:
|
||||
warnings.warn(
|
||||
'Modifying the params dictionary in-place in the function defined in '
|
||||
'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
|
||||
'setting is deprecated. The function must return a new dictionary '
|
||||
'instead.',
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return new_params if new_params is not None else params
|
||||
|
||||
def _load_filter(self, feed_options):
|
||||
# load the item filter if declared else load the default filter class
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
|
|||
"""
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from typing import Generator, Tuple
|
||||
from urllib.parse import urljoin
|
||||
|
|
@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
http_content_type_encoding, resolve_encoding)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
|
|
@ -66,13 +64,6 @@ class TextResponse(Response):
|
|||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
warnings.warn('Response.body_as_unicode() is deprecated, '
|
||||
'please use Response.text instead.',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.text
|
||||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
|
|
|||
|
|
@ -9,45 +9,15 @@ from collections.abc import MutableMapping
|
|||
from copy import deepcopy
|
||||
from pprint import pformat
|
||||
from typing import Dict
|
||||
from warnings import warn
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
class _BaseItem(object_ref):
|
||||
"""
|
||||
Temporary class used internally to avoid the deprecation
|
||||
warning raised by isinstance checks using BaseItem.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class _BaseItemMeta(ABCMeta):
|
||||
def __instancecheck__(cls, instance):
|
||||
if cls is BaseItem:
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
|
||||
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
||||
"""
|
||||
Deprecated, please use :class:`scrapy.item.Item` instead
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
class ItemMeta(_BaseItemMeta):
|
||||
class ItemMeta(ABCMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
.. _metaclass: https://realpython.com/python-metaclasses
|
||||
|
|
@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta):
|
|||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
class Item(MutableMapping, object_ref, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
fields: Dict[str, Field] = {}
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
||||
fields: Dict[str, Field]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __iter__(self):
|
||||
return iter(self._values)
|
||||
|
||||
__hash__ = BaseItem.__hash__
|
||||
__hash__ = object_ref.__hash__
|
||||
|
||||
def keys(self):
|
||||
return self._values.keys()
|
||||
|
|
@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem):
|
|||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import logging
|
||||
import pprint
|
||||
from collections import defaultdict, deque
|
||||
from typing import Callable, Deque, Dict
|
||||
from typing import Callable, Deque, Dict, Optional, cast, Iterable
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
|
@ -9,7 +9,7 @@ from scrapy import Spider
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
|
||||
from scrapy.utils.defer import process_parallel, process_chain
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -21,7 +21,8 @@ class MiddlewareManager:
|
|||
|
||||
def __init__(self, *middlewares):
|
||||
self.middlewares = middlewares
|
||||
self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
|
||||
# Optional because process_spider_output and process_spider_exception can be None
|
||||
self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
|
||||
for mw in middlewares:
|
||||
self._add_middleware(mw)
|
||||
|
||||
|
|
@ -64,14 +65,12 @@ class MiddlewareManager:
|
|||
self.methods['close_spider'].appendleft(mw.close_spider)
|
||||
|
||||
def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_parallel(self.methods[methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_parallel(methods, obj, *args)
|
||||
|
||||
def _process_chain(self, methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain(self.methods[methodname], obj, *args)
|
||||
|
||||
def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
|
||||
return process_chain_both(self.methods[cb_methodname],
|
||||
self.methods[eb_methodname], obj, *args)
|
||||
methods = cast(Iterable[Callable], self.methods[methodname])
|
||||
return process_chain(methods, obj, *args)
|
||||
|
||||
def open_spider(self, spider: Spider) -> Deferred:
|
||||
return self._process_parallel('open_spider', spider)
|
||||
|
|
|
|||
|
|
@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
|
|||
return len(self.attributes)
|
||||
|
||||
def _to_dict(self):
|
||||
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
|
||||
for k, v in self.items()}
|
||||
|
||||
def _get_key(self, key_value):
|
||||
return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
|
||||
else str(key_value))
|
||||
|
||||
def copy_to_dict(self):
|
||||
"""
|
||||
Make a copy of current settings and convert to a dict.
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
|
|||
process_results methods for pre and post-processing purposes.
|
||||
"""
|
||||
|
||||
for row in csviter(response, self.delimiter, self.headers, self.quotechar):
|
||||
for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
|
||||
ret = iterate_spider_output(self.parse_row(response, row))
|
||||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
|
|||
message = (
|
||||
'The -t command line option is deprecated in favor of '
|
||||
'specifying the output format within the output URI. See the '
|
||||
'documentation of the -o and -O options for more information.',
|
||||
'documentation of the -o and -O options for more information.'
|
||||
)
|
||||
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
|
||||
return {output[0]: {'format': output_format}}
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ def _get_handler(settings):
|
|||
return handler
|
||||
|
||||
|
||||
def log_scrapy_info(settings):
|
||||
def log_scrapy_info(settings: Settings) -> None:
|
||||
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
|
||||
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
|
||||
versions = [
|
||||
|
|
@ -152,6 +152,9 @@ def log_scrapy_info(settings):
|
|||
if name != "Scrapy"
|
||||
]
|
||||
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
|
||||
|
||||
|
||||
def log_reactor_info() -> None:
|
||||
from twisted.internet import reactor
|
||||
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
|
||||
from twisted.internet import asyncioreactor
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ from w3lib.html import replace_entities
|
|||
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.item import Item
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
|
||||
|
||||
|
||||
def arg_to_iter(arg):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import sys
|
||||
from contextlib import suppress
|
||||
|
||||
from twisted.internet import asyncioreactor, error
|
||||
|
|
@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None):
|
|||
reactor_class = load_object(reactor_path)
|
||||
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
|
||||
with suppress(error.ReactorAlreadyInstalledError):
|
||||
if sys.version_info >= (3, 8) and sys.platform == "win32":
|
||||
policy = asyncio.get_event_loop_policy()
|
||||
if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
if event_loop_path is not None:
|
||||
event_loop_class = load_object(event_loop_path)
|
||||
event_loop = event_loop_class()
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ This module provides some useful functions for working with
|
|||
scrapy.http.Response objects
|
||||
"""
|
||||
import os
|
||||
import webbrowser
|
||||
import re
|
||||
import tempfile
|
||||
import webbrowser
|
||||
from typing import Any, Callable, Iterable, Optional, Tuple, Union
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
|
|
@ -13,6 +14,7 @@ from scrapy.http.response import Response
|
|||
|
||||
from twisted.web import http
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from w3lib import html
|
||||
|
||||
|
||||
|
|
@ -50,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
|
|||
return f'{status_int} {to_unicode(message)}'
|
||||
|
||||
|
||||
@deprecated
|
||||
def response_httprepr(response: Response) -> bytes:
|
||||
"""Return raw HTTP representation (as bytes) of the given response. This
|
||||
is provided only for reference, since it's not the exact stream of bytes
|
||||
|
|
@ -80,8 +83,9 @@ def open_in_browser(
|
|||
body = response.body
|
||||
if isinstance(response, HtmlResponse):
|
||||
if b'<base' not in body:
|
||||
repl = f'<head><base href="{response.url}">'
|
||||
body = body.replace(b'<head>', to_bytes(repl))
|
||||
repl = fr'\1<base href="{response.url}">'
|
||||
body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
|
||||
body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
|
||||
ext = '.html'
|
||||
elif isinstance(response, TextResponse):
|
||||
ext = '.txt'
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import asyncio
|
||||
import sys
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if sys.version_info >= (3, 8) and sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
|
||||
import scrapy
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class AsyncioReactorSpider(scrapy.Spider):
|
||||
name = 'asyncio_reactor'
|
||||
custom_settings = {
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
|
||||
|
||||
process = CrawlerProcess()
|
||||
process.crawl(AsyncioReactorSpider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class SelectReactorSpider(scrapy.Spider):
|
||||
name = 'select_reactor'
|
||||
custom_settings = {
|
||||
"TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
|
||||
}
|
||||
|
||||
|
||||
class AsyncioReactorSpider(scrapy.Spider):
|
||||
name = 'asyncio_reactor'
|
||||
custom_settings = {
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
|
||||
|
||||
process = CrawlerProcess()
|
||||
process.crawl(SelectReactorSpider)
|
||||
process.crawl(AsyncioReactorSpider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class AsyncioReactorSpider1(scrapy.Spider):
|
||||
name = 'asyncio_reactor1'
|
||||
custom_settings = {
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
|
||||
class AsyncioReactorSpider2(scrapy.Spider):
|
||||
name = 'asyncio_reactor2'
|
||||
custom_settings = {
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
}
|
||||
|
||||
|
||||
process = CrawlerProcess()
|
||||
process.crawl(AsyncioReactorSpider1)
|
||||
process.crawl(AsyncioReactorSpider2)
|
||||
process.start()
|
||||
|
|
@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl
|
|||
from twisted.internet.task import deferLater
|
||||
from twisted.names import dns, error
|
||||
from twisted.names.server import DNSServerFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper, Resource
|
||||
from twisted.web import resource, server
|
||||
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
|
||||
from twisted.web.static import File
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.util import redirectTo
|
||||
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
|
@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None):
|
|||
return default
|
||||
|
||||
|
||||
class LeafResource(Resource):
|
||||
# most of the following resources are copied from twisted.web.test.test_webclient
|
||||
class ForeverTakingResource(resource.Resource):
|
||||
"""
|
||||
L{ForeverTakingResource} is a resource which never finishes responding
|
||||
to requests.
|
||||
"""
|
||||
|
||||
def __init__(self, write=False):
|
||||
resource.Resource.__init__(self)
|
||||
self._write = write
|
||||
|
||||
def render(self, request):
|
||||
if self._write:
|
||||
request.write(b"some bytes")
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
||||
class ErrorResource(resource.Resource):
|
||||
def render(self, request):
|
||||
request.setResponseCode(401)
|
||||
if request.args.get(b"showlength"):
|
||||
request.setHeader(b"content-length", b"0")
|
||||
return b""
|
||||
|
||||
|
||||
class NoLengthResource(resource.Resource):
|
||||
def render(self, request):
|
||||
return b"nolength"
|
||||
|
||||
|
||||
class HostHeaderResource(resource.Resource):
|
||||
"""
|
||||
A testing resource which renders itself as the value of the host header
|
||||
from the request.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
return request.requestHeaders.getRawHeaders(b"host")[0]
|
||||
|
||||
|
||||
class PayloadResource(resource.Resource):
|
||||
"""
|
||||
A testing resource which renders itself as the contents of the request body
|
||||
as long as the request body is 100 bytes long, otherwise which renders
|
||||
itself as C{"ERROR"}.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
data = request.content.read()
|
||||
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
|
||||
if len(data) != 100 or int(contentLength) != 100:
|
||||
return b"ERROR"
|
||||
return data
|
||||
|
||||
|
||||
class BrokenDownloadResource(resource.Resource):
|
||||
def render(self, request):
|
||||
# only sends 3 bytes even though it claims to send 5
|
||||
request.setHeader(b"content-length", b"5")
|
||||
request.write(b"abc")
|
||||
return b""
|
||||
|
||||
|
||||
class LeafResource(resource.Resource):
|
||||
|
||||
isLeaf = True
|
||||
|
||||
|
|
@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource):
|
|||
return request.content.read()
|
||||
|
||||
|
||||
class Root(Resource):
|
||||
class Root(resource.Resource):
|
||||
|
||||
def __init__(self):
|
||||
Resource.__init__(self)
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"status", Status())
|
||||
self.putChild(b"follow", Follow())
|
||||
self.putChild(b"delay", Delay())
|
||||
|
|
@ -187,7 +249,7 @@ class Root(Resource):
|
|||
self.putChild(b"raw", Raw())
|
||||
self.putChild(b"echo", Echo())
|
||||
self.putChild(b"payload", PayloadResource())
|
||||
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
|
||||
try:
|
||||
from tests import tests_datadir
|
||||
|
|
|
|||
|
|
@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase):
|
|||
settingsdict = json.loads(settingsstr)
|
||||
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
|
||||
self.assertEqual(200, settingsdict[EXT_PATH])
|
||||
|
||||
def test_pathlib_path_as_feeds_key(self):
|
||||
self.assertEqual(self._execute('settings', '--get', 'FEEDS'),
|
||||
json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
from pathlib import Path
|
||||
|
||||
EXTENSIONS = {
|
||||
'tests.test_cmdline.extensions.TestExtension': 0,
|
||||
}
|
||||
|
||||
TEST1 = 'default'
|
||||
|
||||
FEEDS = {
|
||||
Path('items.csv'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ import scrapy
|
|||
|
||||
class CheckSpider(scrapy.Spider):
|
||||
name = '{self.spider_name}'
|
||||
start_urls = ['http://example.com']
|
||||
start_urls = ['http://toscrape.com']
|
||||
|
||||
def parse(self, response, **cb_kwargs):
|
||||
\"\"\"
|
||||
@url http://example.com
|
||||
@url http://toscrape.com
|
||||
{contracts}
|
||||
\"\"\"
|
||||
{parse_def}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import os
|
||||
import argparse
|
||||
from os.path import join, abspath, isfile, exists
|
||||
from twisted.internet import defer
|
||||
from scrapy.commands import parse
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.testsite import SiteTest
|
||||
from scrapy.utils.testproc import ProcessTest
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
|
|||
content = '[\n{},\n{"foo": "bar"}\n]'
|
||||
with open(file_path, 'r') as f:
|
||||
self.assertEqual(f.read(), content)
|
||||
|
||||
def test_parse_add_options(self):
|
||||
command = parse.Command()
|
||||
command.settings = Settings()
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='scrapy', formatter_class=argparse.HelpFormatter,
|
||||
conflict_handler='resolve', prefix_chars='-'
|
||||
)
|
||||
command.add_options(parser)
|
||||
namespace = parser.parse_args(
|
||||
['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
|
||||
)
|
||||
self.assertTrue(namespace.nolinks)
|
||||
self.assertEqual(namespace.depth, 2)
|
||||
self.assertEqual(namespace.spider, self.spider_name)
|
||||
self.assertTrue(namespace.verbose)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import inspect
|
||||
import json
|
||||
import optparse
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
|
|
@ -23,7 +23,7 @@ from twisted.python.versions import Version
|
|||
from twisted.trial import unittest
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
|
||||
from scrapy.commands.startproject import IGNORE
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
|
|||
def setUp(self):
|
||||
self.command = ScrapyCommand()
|
||||
self.command.settings = Settings()
|
||||
self.parser = optparse.OptionParser(
|
||||
formatter=optparse.TitledHelpFormatter(),
|
||||
conflict_handler='resolve',
|
||||
)
|
||||
self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
|
||||
conflict_handler='resolve')
|
||||
self.command.add_options(self.parser)
|
||||
|
||||
def test_settings_json_string(self):
|
||||
feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
|
||||
opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
|
||||
opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
|
||||
self.command.process_options(args, opts)
|
||||
self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
|
||||
self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
|
||||
|
||||
def test_help_formatter(self):
|
||||
formatter = ScrapyHelpFormatter(prog='scrapy')
|
||||
part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
|
||||
'\n', 'optional arguments:\n', '\n', 'Global Options:\n']
|
||||
self.assertEqual(
|
||||
formatter._join_parts(part_strings),
|
||||
('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'
|
||||
'Optional Arguments\n==================\n\n'
|
||||
'Global Options\n--------------\n')
|
||||
)
|
||||
|
||||
|
||||
class ProjectTest(unittest.TestCase):
|
||||
project_name = 'testproject'
|
||||
|
|
@ -674,9 +683,6 @@ class MySpider(scrapy.Spider):
|
|||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.py", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_asyncio_enabled_true(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[
|
||||
'-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
|
||||
|
|
@ -699,15 +705,15 @@ class MySpider(scrapy.Spider):
|
|||
])
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_custom_asyncio_loop_enabled_false(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[
|
||||
'-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
|
||||
])
|
||||
import asyncio
|
||||
loop = asyncio.new_event_loop()
|
||||
if sys.platform != 'win32':
|
||||
loop = asyncio.new_event_loop()
|
||||
else:
|
||||
loop = asyncio.SelectorEventLoop()
|
||||
self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
|
||||
|
||||
def test_output(self):
|
||||
|
|
@ -765,6 +771,7 @@ class MySpider(scrapy.Spider):
|
|||
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
|
||||
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
|
||||
|
||||
spider_filename = 'myspider.pyw'
|
||||
|
|
@ -777,35 +784,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
|
|||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.pyw", log)
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_run_good_spider(self):
|
||||
super().test_run_good_spider()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider(self):
|
||||
super().test_runspider()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_dnscache_disabled(self):
|
||||
super().test_runspider_dnscache_disabled()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_log_level(self):
|
||||
super().test_runspider_log_level()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_log_short_names(self):
|
||||
super().test_runspider_log_short_names()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_no_spider_found(self):
|
||||
super().test_runspider_no_spider_found()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_output(self):
|
||||
super().test_output()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_overwrite_output(self):
|
||||
super().test_overwrite_output()
|
||||
|
||||
|
|
@ -822,6 +821,21 @@ class BenchCommandTest(CommandTest):
|
|||
self.assertNotIn('Unhandled Error', log)
|
||||
|
||||
|
||||
class ViewCommandTest(CommandTest):
|
||||
|
||||
def test_methods(self):
|
||||
command = view.Command()
|
||||
command.settings = Settings()
|
||||
parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
|
||||
formatter_class=ScrapyHelpFormatter,
|
||||
conflict_handler='resolve')
|
||||
command.add_options(parser)
|
||||
self.assertEqual(command.short_desc(),
|
||||
"Open URL in browser, as seen by Scrapy")
|
||||
self.assertIn("URL using the Scrapy downloader and show its",
|
||||
command.long_desc())
|
||||
|
||||
|
||||
class CrawlCommandTest(CommandTest):
|
||||
|
||||
def crawl(self, code, args=()):
|
||||
|
|
|
|||
|
|
@ -4,10 +4,8 @@ import platform
|
|||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
from unittest import skipIf
|
||||
|
||||
from pytest import raises, mark
|
||||
from testfixtures import LogCapture
|
||||
from twisted import version as twisted_version
|
||||
from twisted.internet import defer
|
||||
from twisted.python.versions import Version
|
||||
|
|
@ -271,6 +269,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
|
||||
self.assertEqual(runner.bootstrap_failed, True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawler_runner_asyncio_enabled_true(self):
|
||||
if self.reactor_pytest == 'asyncio':
|
||||
CrawlerRunner(settings={
|
||||
|
|
@ -279,35 +278,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
else:
|
||||
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
|
||||
with self.assertRaisesRegex(Exception, msg):
|
||||
CrawlerRunner(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_crawler_process_asyncio_enabled_true(self):
|
||||
with LogCapture(level=logging.DEBUG) as log:
|
||||
if self.reactor_pytest == 'asyncio':
|
||||
runner = CrawlerProcess(settings={
|
||||
runner = CrawlerRunner(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
})
|
||||
yield runner.crawl(NoRequestsSpider)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
|
||||
else:
|
||||
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
|
||||
with self.assertRaisesRegex(Exception, msg):
|
||||
runner = CrawlerProcess(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
})
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawler_process_asyncio_enabled_false(self):
|
||||
runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
|
||||
with LogCapture(level=logging.DEBUG) as log:
|
||||
yield runner.crawl(NoRequestsSpider)
|
||||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
|
||||
|
||||
|
||||
class ScriptRunnerMixin:
|
||||
|
|
@ -328,17 +302,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_asyncio_enabled_no_reactor(self):
|
||||
log = self.run_script('asyncio_enabled_no_reactor.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_asyncio_enabled_reactor(self):
|
||||
log = self.run_script('asyncio_enabled_reactor.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
|
|
@ -377,14 +345,26 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_reactor_asyncio(self):
|
||||
log = self.run_script("twisted_reactor_asyncio.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
def test_reactor_asyncio_custom_settings(self):
|
||||
log = self.run_script("twisted_reactor_custom_settings.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
def test_reactor_asyncio_custom_settings_same(self):
|
||||
log = self.run_script("twisted_reactor_custom_settings_same.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
def test_reactor_asyncio_custom_settings_conflict(self):
|
||||
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
|
||||
self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
|
||||
self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
|
||||
@mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
|
||||
|
|
@ -404,9 +384,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
self.assertIn("async pipeline opened!", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_default_loop_asyncio_deferred_signal(self):
|
||||
log = self.run_script("asyncio_deferred_signal.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Optional, Type
|
||||
from unittest import mock
|
||||
|
|
@ -14,8 +15,6 @@ from twisted.trial import unittest
|
|||
from twisted.web import resource, server, static, util
|
||||
from twisted.web._newclient import ResponseFailed
|
||||
from twisted.web.http import _DataLoss
|
||||
from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource,
|
||||
NoLengthResource, PayloadResource)
|
||||
from w3lib.url import path_to_file_uri
|
||||
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
|
@ -33,7 +32,15 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.test import get_crawler, skip_if_no_boto
|
||||
from tests.mockserver import MockServer, ssl_context_factory, Echo
|
||||
from tests.mockserver import (
|
||||
Echo,
|
||||
ForeverTakingResource,
|
||||
HostHeaderResource,
|
||||
MockServer,
|
||||
NoLengthResource,
|
||||
PayloadResource,
|
||||
ssl_context_factory,
|
||||
)
|
||||
from tests.spiders import SingleRequestSpider
|
||||
|
||||
|
||||
|
|
@ -287,6 +294,12 @@ class HttpTestCase(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_timeout_download_from_spider_nodata_rcvd(self):
|
||||
if self.reactor_pytest == "asyncio" and sys.platform == "win32":
|
||||
# https://twistedmatrix.com/trac/ticket/10279
|
||||
raise unittest.SkipTest(
|
||||
"This test produces DirtyReactorAggregateError on Windows with asyncio"
|
||||
)
|
||||
|
||||
# client connects but no data is received
|
||||
spider = Spider('foo')
|
||||
meta = {'download_timeout': 0.5}
|
||||
|
|
@ -296,6 +309,11 @@ class HttpTestCase(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_timeout_download_from_spider_server_hangs(self):
|
||||
if self.reactor_pytest == "asyncio" and sys.platform == "win32":
|
||||
# https://twistedmatrix.com/trac/ticket/10279
|
||||
raise unittest.SkipTest(
|
||||
"This test produces DirtyReactorAggregateError on Windows with asyncio"
|
||||
)
|
||||
# client connects, server send headers and some body bytes but hangs
|
||||
spider = Spider('foo')
|
||||
meta = {'download_timeout': 0.5}
|
||||
|
|
@ -1055,6 +1073,10 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
class FTPTestCase(BaseFTPTestCase):
|
||||
|
||||
def test_invalid_credentials(self):
|
||||
if self.reactor_pytest == "asyncio" and sys.platform == "win32":
|
||||
raise unittest.SkipTest(
|
||||
"This test produces DirtyReactorAggregateError on Windows with asyncio"
|
||||
)
|
||||
from twisted.protocols.ftp import ConnectionLost
|
||||
|
||||
meta = dict(self.req_meta)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
from itertools import product
|
||||
from unittest import TestCase
|
||||
|
||||
from scrapy.downloadermiddlewares.stats import DownloaderStats
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase):
|
|||
self.mw.process_response(self.req, self.res, self.spider)
|
||||
self.assertStatsEqual('downloader/response_count', 1)
|
||||
|
||||
def test_response_len(self):
|
||||
body = (b'', b'not_empty') # empty/notempty body
|
||||
headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h
|
||||
test_responses = [ # form test responses with all combinations of body/headers
|
||||
Response(
|
||||
url='scrapytest.org',
|
||||
status=200,
|
||||
body=r[0],
|
||||
headers=r[1]
|
||||
)
|
||||
for r in product(body, headers)
|
||||
]
|
||||
for test_response in test_responses:
|
||||
self.crawler.stats.set_value('downloader/response_bytes', 0)
|
||||
self.mw.process_response(self.req, test_response, self.spider)
|
||||
self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
|
||||
|
||||
def test_process_exception(self):
|
||||
self.mw.process_exception(self.req, MyException(), self.spider)
|
||||
self.assertStatsEqual('downloader/exception_count', 1)
|
||||
|
|
|
|||
|
|
@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
|||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class URIParamsTest:
|
||||
|
||||
spider_name = "uri_params_spider"
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def test_default(self):
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_none(self):
|
||||
def uri_params(params, spider):
|
||||
pass
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
'Modifying the params dictionary in-place in the '
|
||||
'function defined in the FEED_URI_PARAMS setting or '
|
||||
'in the uri_params key of the FEEDS setting is '
|
||||
'deprecated. The function must return a new '
|
||||
'dictionary instead.'
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_empty_dict(self):
|
||||
def uri_params(params, spider):
|
||||
return {}
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
with self.assertRaises(KeyError):
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
def test_params_as_is(self):
|
||||
def uri_params(params, spider):
|
||||
return params
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(name)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
def test_custom_param(self):
|
||||
def uri_params(params, spider):
|
||||
return {**params, 'foo': self.spider_name}
|
||||
|
||||
settings = self.build_settings(
|
||||
uri='file:///tmp/%(foo)s',
|
||||
uri_params=uri_params,
|
||||
)
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider(self.spider_name)
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(
|
||||
str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning
|
||||
)
|
||||
self.assertEqual(messages, tuple())
|
||||
|
||||
self.assertEqual(
|
||||
feed_exporter.slots[0].uri,
|
||||
f'file:///tmp/{self.spider_name}'
|
||||
)
|
||||
|
||||
|
||||
class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
extra_settings = {}
|
||||
if uri_params:
|
||||
extra_settings['FEED_URI_PARAMS'] = uri_params
|
||||
return {
|
||||
'FEED_URI': uri,
|
||||
**extra_settings,
|
||||
}
|
||||
|
||||
|
||||
class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
|
||||
|
||||
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
|
||||
options = {
|
||||
'format': 'jl',
|
||||
}
|
||||
if uri_params:
|
||||
options['uri_params'] = uri_params
|
||||
return {
|
||||
'FEEDS': {
|
||||
uri: options,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
import unittest
|
||||
from unittest import mock
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from w3lib.encoding import resolve_encoding
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
|
||||
XmlResponse, Headers)
|
||||
from scrapy.selector import Selector
|
||||
|
|
@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase):
|
|||
assert isinstance(response.text, str)
|
||||
self._assert_response_encoding(response, encoding)
|
||||
self.assertEqual(response.body, body_bytes)
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertEqual(response.body_as_unicode(), body_unicode)
|
||||
self.assertEqual(response.text, body_unicode)
|
||||
|
||||
def _assert_response_encoding(self, response, encoding):
|
||||
|
|
@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest):
|
|||
original_string = unicode_string.encode('cp1251')
|
||||
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
|
||||
|
||||
# check body_as_unicode
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertTrue(isinstance(r1.body_as_unicode(), str))
|
||||
self.assertEqual(r1.body_as_unicode(), unicode_string)
|
||||
|
||||
# check response.text
|
||||
self.assertTrue(isinstance(r1.text, str))
|
||||
self.assertEqual(r1.text, unicode_string)
|
||||
|
|
@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest):
|
|||
with self.assertRaises(ValueError):
|
||||
response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
|
||||
|
||||
def test_body_as_unicode_deprecation_warning(self):
|
||||
with catch_warnings(record=True) as warnings:
|
||||
r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
|
||||
self.assertEqual(r1.body_as_unicode(), 'Hello')
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
def test_json_response(self):
|
||||
json_body = b"""{"ip": "109.187.217.200"}"""
|
||||
json_response = self.response_class("http://www.example.com", body=json_body)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
import unittest
|
||||
from unittest import mock
|
||||
from warnings import catch_warnings, filterwarnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
|
||||
from scrapy.item import ABCMeta, Field, Item, ItemMeta
|
||||
|
||||
|
||||
class ItemTest(unittest.TestCase):
|
||||
|
|
@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase):
|
|||
item['tags'].append('tag2')
|
||||
assert item['tags'] != copied_item['tags']
|
||||
|
||||
def test_dictitem_deprecation_warning(self):
|
||||
"""Make sure the DictItem deprecation warning is not issued for
|
||||
Item"""
|
||||
with catch_warnings(record=True) as warnings:
|
||||
Item()
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
SubclassedItem()
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
|
||||
class ItemMetaTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -303,94 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase):
|
|||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class DictItemTest(unittest.TestCase):
|
||||
|
||||
def test_deprecation_warning(self):
|
||||
with catch_warnings(record=True) as warnings:
|
||||
DictItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
with catch_warnings(record=True) as warnings:
|
||||
class SubclassedDictItem(DictItem):
|
||||
pass
|
||||
SubclassedDictItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
class BaseItemTest(unittest.TestCase):
|
||||
|
||||
def test_isinstance_check(self):
|
||||
|
||||
class SubclassedBaseItem(BaseItem):
|
||||
pass
|
||||
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
|
||||
with catch_warnings():
|
||||
filterwarnings("ignore", category=ScrapyDeprecationWarning)
|
||||
self.assertTrue(isinstance(BaseItem(), BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
|
||||
self.assertTrue(isinstance(Item(), BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedItem(), BaseItem))
|
||||
|
||||
# make sure internal checks using private _BaseItem class succeed
|
||||
self.assertTrue(isinstance(BaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(Item(), _BaseItem))
|
||||
self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
|
||||
|
||||
def test_deprecation_warning(self):
|
||||
"""
|
||||
Make sure deprecation warnings are logged whenever BaseItem is used,
|
||||
either instantiated or in an isinstance check
|
||||
"""
|
||||
with catch_warnings(record=True) as warnings:
|
||||
BaseItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
|
||||
class SubclassedBaseItem(BaseItem):
|
||||
pass
|
||||
|
||||
SubclassedBaseItem()
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
self.assertFalse(isinstance("foo", BaseItem))
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
self.assertTrue(isinstance(BaseItem(), BaseItem))
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
||||
class ItemNoDeprecationWarningTest(unittest.TestCase):
|
||||
def test_no_deprecation_warning(self):
|
||||
"""
|
||||
Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
|
||||
"""
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
|
||||
with catch_warnings(record=True) as warnings:
|
||||
Item()
|
||||
SubclassedItem()
|
||||
_BaseItem()
|
||||
self.assertFalse(isinstance("foo", _BaseItem))
|
||||
self.assertFalse(isinstance("foo", Item))
|
||||
self.assertFalse(isinstance("foo", SubclassedItem))
|
||||
self.assertTrue(isinstance(_BaseItem(), _BaseItem))
|
||||
self.assertTrue(isinstance(Item(), Item))
|
||||
self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from scrapy.spiders import (
|
|||
)
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
class SpiderTest(unittest.TestCase):
|
||||
|
|
@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest):
|
|||
|
||||
spider_class = CSVFeedSpider
|
||||
|
||||
def test_parse_rows(self):
|
||||
body = get_testdata('feeds', 'feed-sample6.csv')
|
||||
response = Response("http://example.org/dummy.csv", body=body)
|
||||
|
||||
class _CrawlSpider(self.spider_class):
|
||||
name = "test"
|
||||
delimiter = ","
|
||||
quotechar = "'"
|
||||
|
||||
def parse_row(self, response, row):
|
||||
return row
|
||||
|
||||
spider = _CrawlSpider()
|
||||
rows = list(spider.parse_rows(response))
|
||||
assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'}
|
||||
assert len(rows) == 4
|
||||
|
||||
|
||||
class CrawlSpiderTest(SpiderTest):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
import platform
|
||||
import sys
|
||||
from unittest import skipIf, TestCase
|
||||
from unittest import TestCase
|
||||
|
||||
from pytest import mark
|
||||
|
||||
|
|
@ -14,9 +12,6 @@ class AsyncioTest(TestCase):
|
|||
# the result should depend only on the pytest --reactor argument
|
||||
self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_install_asyncio_reactor(self):
|
||||
# this should do nothing
|
||||
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
|
||||
|
|
|
|||
|
|
@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
|
|||
self.assertEqual(response_status_message(200), '200 OK')
|
||||
self.assertEqual(response_status_message(404), '404 Not Found')
|
||||
self.assertEqual(response_status_message(573), "573 Unknown Status")
|
||||
|
||||
def test_inject_base_url(self):
|
||||
url = "http://www.example.com"
|
||||
|
||||
def check_base_url(burl):
|
||||
path = urlparse(burl).path
|
||||
if not os.path.exists(path):
|
||||
path = burl.replace('file://', '')
|
||||
with open(path, "rb") as f:
|
||||
bbody = f.read()
|
||||
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
|
||||
return True
|
||||
|
||||
r1 = HtmlResponse(url, body=b"""
|
||||
<html>
|
||||
<head><title>Dummy</title></head>
|
||||
<body><p>Hello world.</p></body>
|
||||
</html>""")
|
||||
r2 = HtmlResponse(url, body=b"""
|
||||
<html>
|
||||
<head id="foo"><title>Dummy</title></head>
|
||||
<body>Hello world.</body>
|
||||
</html>""")
|
||||
r3 = HtmlResponse(url, body=b"""
|
||||
<html>
|
||||
<head><title>Dummy</title></head>
|
||||
<body>
|
||||
<header>Hello header</header>
|
||||
<p>Hello world.</p>
|
||||
</body>
|
||||
</html>""")
|
||||
r4 = HtmlResponse(url, body=b"""
|
||||
<html>
|
||||
<!-- <head>Dummy comment</head> -->
|
||||
<head><title>Dummy</title></head>
|
||||
<body><p>Hello world.</p></body>
|
||||
</html>""")
|
||||
r5 = HtmlResponse(url, body=b"""
|
||||
<html>
|
||||
<!--[if IE]>
|
||||
<head><title>IE head</title></head>
|
||||
<![endif]-->
|
||||
<!--[if !IE]>-->
|
||||
<head><title>Standard head</title></head>
|
||||
<!--<![endif]-->
|
||||
<body><p>Hello world.</p></body>
|
||||
</html>""")
|
||||
|
||||
assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
|
||||
assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
|
||||
assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
|
||||
assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
|
||||
assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"
|
||||
|
|
|
|||
|
|
@ -21,14 +21,6 @@ except ImportError:
|
|||
from twisted.python.filepath import FilePath
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.web.test.test_webclient import (
|
||||
ForeverTakingResource,
|
||||
ErrorResource,
|
||||
NoLengthResource,
|
||||
HostHeaderResource,
|
||||
PayloadResource,
|
||||
BrokenDownloadResource,
|
||||
)
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
|
||||
|
|
@ -36,7 +28,15 @@ from scrapy.http import Request, Headers
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from tests.mockserver import ssl_context_factory
|
||||
from tests.mockserver import (
|
||||
BrokenDownloadResource,
|
||||
ErrorResource,
|
||||
ForeverTakingResource,
|
||||
HostHeaderResource,
|
||||
NoLengthResource,
|
||||
PayloadResource,
|
||||
ssl_context_factory,
|
||||
)
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
|
||||
|
|
|
|||
Loading…
Reference in New Issue