mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into flake8-max-line-length
This commit is contained in:
Eugenio Lacuesta
commit
25eeb77ba6
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.0.0
|
||||
current_version = 2.1.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
22
.travis.yml
22
.travis.yml
|
|
@ -11,25 +11,23 @@ matrix:
|
|||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- env: TOXENV=py35
|
||||
python: 3.5
|
||||
- python: 3.5
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5
|
||||
- env: TOXENV=py35-asyncio
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2
|
||||
- env: TOXENV=py36
|
||||
python: 3.6
|
||||
- env: TOXENV=py37
|
||||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
- python: 3.6
|
||||
- python: 3.7
|
||||
- env: PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-asyncio
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
|
|
@ -62,4 +60,4 @@ deploy:
|
|||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
|
|||
147
docs/news.rst
147
docs/news.rst
|
|
@ -3,6 +3,153 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.1.0:
|
||||
|
||||
Scrapy 2.1.0 (2020-04-24)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* New :setting:`FEEDS` setting to export to multiple feeds
|
||||
* New :attr:`Response.ip_address <scrapy.http.Response.ip_address>` attribute
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* :exc:`AssertionError` exceptions triggered by :ref:`assert <assert>`
|
||||
statements have been replaced by new exception types, to support running
|
||||
Python in optimized mode (see :option:`-O`) without changing Scrapy’s
|
||||
behavior in any unexpected ways.
|
||||
|
||||
If you catch an :exc:`AssertionError` exception from Scrapy, update your
|
||||
code to catch the corresponding new exception.
|
||||
|
||||
(:issue:`4440`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use
|
||||
:setting:`SCHEDULER_DEBUG` instead (:issue:`4385`)
|
||||
|
||||
* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use
|
||||
:setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`)
|
||||
|
||||
* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware`
|
||||
middleware has been removed, including the entire
|
||||
:class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers
|
||||
work out of the box (:issue:`4431`)
|
||||
|
||||
* The ``spiders`` property has been removed from
|
||||
:class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader
|
||||
<scrapy.crawler.CrawlerRunner.spider_loader>` or instantiate
|
||||
:setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`)
|
||||
|
||||
* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode``
|
||||
classes have been removed from :mod:`scrapy.utils.datatypes`
|
||||
(:issue:`4400`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in
|
||||
favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`,
|
||||
:issue:`4507`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds
|
||||
with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`)
|
||||
|
||||
* The :command:`crawl` and :command:`runspider` commands now support multiple
|
||||
``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`)
|
||||
|
||||
* The :command:`crawl` and :command:`runspider` commands now support
|
||||
specifying an output format by appending ``:<format>`` to the output file
|
||||
(:issue:`1336`, :issue:`3858`, :issue:`4507`)
|
||||
|
||||
* The new :attr:`Response.ip_address <scrapy.http.Response.ip_address>`
|
||||
attribute gives access to the IP address that originated a response
|
||||
(:issue:`3903`, :issue:`3940`)
|
||||
|
||||
* A warning is now issued when a value in
|
||||
:attr:`~scrapy.spiders.Spider.allowed_domains` includes a port
|
||||
(:issue:`50`, :issue:`3198`, :issue:`4413`)
|
||||
|
||||
* Zsh completion now excludes used option aliases from the completion list
|
||||
(:issue:`4438`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :ref:`Request serialization <request-serialization>` no longer breaks for
|
||||
callbacks that are spider attributes which are assigned a function with a
|
||||
different name (:issue:`4500`)
|
||||
|
||||
* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer
|
||||
cause a :exc:`TypeError` exception (:issue:`4410`)
|
||||
|
||||
* Zsh completion no longer allows options after arguments (:issue:`4438`)
|
||||
|
||||
* zope.interface 5.0.0 and later versions are now supported
|
||||
(:issue:`4447`, :issue:`4448`)
|
||||
|
||||
* :meth:`Spider.make_requests_from_url
|
||||
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
|
||||
1.4.0, now issues a warning when used (:issue:`4412`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Improved the documentation about signals that allow their handlers to
|
||||
return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`,
|
||||
:issue:`4390`)
|
||||
|
||||
* Our PyPI entry now includes links for our documentation, our source code
|
||||
repository and our issue tracker (:issue:`4456`)
|
||||
|
||||
* Covered the `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_
|
||||
service in the documentation (:issue:`4206`, :issue:`4455`)
|
||||
|
||||
* Removed references to the Guppy library, which only works in Python 2
|
||||
(:issue:`4285`, :issue:`4343`)
|
||||
|
||||
* Extended use of InterSphinx to link to Python 3 documentation
|
||||
(:issue:`4444`, :issue:`4445`)
|
||||
|
||||
* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`,
|
||||
:issue:`4496`, :issue:`4503`)
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Removed warnings about using old, removed settings (:issue:`4404`)
|
||||
|
||||
* Removed a warning about importing
|
||||
:class:`~twisted.internet.testing.StringTransport` from
|
||||
``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`)
|
||||
|
||||
* Removed outdated Debian package build files (:issue:`4384`)
|
||||
|
||||
* Removed :class:`object` usage as a base class (:issue:`4430`)
|
||||
|
||||
* Removed code that added support for old versions of Twisted that we no
|
||||
longer support (:issue:`4472`)
|
||||
|
||||
* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`,
|
||||
:issue:`4481`)
|
||||
|
||||
* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`,
|
||||
:issue:`4446`, :issue:`4489`)
|
||||
|
||||
|
||||
.. _release-2.0.1:
|
||||
|
||||
Scrapy 2.0.1 (2020-03-18)
|
||||
|
|
|
|||
|
|
@ -7,10 +7,6 @@ Coroutines
|
|||
Scrapy has :ref:`partial support <coroutine-support>` for the
|
||||
:ref:`coroutine syntax <async>`.
|
||||
|
||||
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
|
||||
versions may introduce related API and behavior changes without a
|
||||
deprecation period or warning.
|
||||
|
||||
.. _coroutine-support:
|
||||
|
||||
Supported callables
|
||||
|
|
|
|||
|
|
@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
|
|||
...
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
'FEED_FORMAT': 'json',
|
||||
'FEED_URI': 'items.json'
|
||||
"FEEDS": {
|
||||
"items.json": {"format": "json"},
|
||||
},
|
||||
})
|
||||
|
||||
process.crawl(MySpider)
|
||||
|
|
|
|||
|
|
@ -619,6 +619,9 @@ Response objects
|
|||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
||||
.. versionadded:: 2.1.0
|
||||
The ``ip_address`` parameter.
|
||||
|
||||
.. attribute:: Response.url
|
||||
|
||||
A string containing the URL of the response.
|
||||
|
|
@ -710,6 +713,8 @@ Response objects
|
|||
|
||||
.. attribute:: Response.ip_address
|
||||
|
||||
.. versionadded:: 2.1.0
|
||||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
|
|
|
|||
|
|
@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
|
|||
.. note::
|
||||
|
||||
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
|
||||
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
|
||||
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
|
||||
accordingly, i.e. to
|
||||
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
unless you really want to use HTTP/1.0 and override
|
||||
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
|
||||
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
|
||||
|
||||
|
|
@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
|
|||
Scrapy also has another context factory class that you can set,
|
||||
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
|
||||
which uses the platform's certificates to validate remote endpoints.
|
||||
**This is only available if you use Twisted>=14.0.**
|
||||
|
||||
If you do use a custom ContextFactory, make sure its ``__init__`` method
|
||||
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
|
||||
|
|
@ -494,10 +492,6 @@ This setting must be one of these string values:
|
|||
- ``'TLSv1.2'``: forces TLS version 1.2
|
||||
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
|
||||
or above (Twisted>=14.0 if you can).
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
|
||||
|
||||
|
|
@ -660,8 +654,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_maxsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_WARNSIZE
|
||||
|
||||
DOWNLOAD_WARNSIZE
|
||||
|
|
@ -679,8 +671,6 @@ If you want to disable it set to 0.
|
|||
spider attribute and per-request using :reqmeta:`download_warnsize`
|
||||
Request.meta key.
|
||||
|
||||
This feature needs Twisted >= 11.1.
|
||||
|
||||
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS
|
||||
|
|
|
|||
|
|
@ -156,6 +156,17 @@ First, we launch the shell::
|
|||
|
||||
scrapy shell 'https://scrapy.org' --nolog
|
||||
|
||||
.. note::
|
||||
|
||||
Remember to always enclose URLs in quotes when running the Scrapy shell from
|
||||
the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
|
||||
will not work.
|
||||
|
||||
On Windows, use double quotes instead::
|
||||
|
||||
scrapy shell "https://scrapy.org" --nolog
|
||||
|
||||
|
||||
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
|
||||
list of available objects and useful shortcuts (you'll notice that these lines
|
||||
all start with the ``[s]`` prefix)::
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
|
|||
You can connect to signals (or send your own) through the
|
||||
:ref:`topics-api-signals`.
|
||||
|
||||
Here is a simple example showing how you can catch signals and perform some action:
|
||||
::
|
||||
Here is a simple example showing how you can catch signals and perform some action::
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy import Spider
|
||||
|
|
@ -52,9 +51,45 @@ Deferred signal handlers
|
|||
========================
|
||||
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, see the :ref:`topics-signals-ref` below to know
|
||||
which ones.
|
||||
objects from their handlers, allowing you to run asynchronous code that
|
||||
does not block Scrapy. If a signal handler returns a
|
||||
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
|
||||
:class:`~twisted.internet.defer.Deferred` to fire.
|
||||
|
||||
Let's take an example::
|
||||
|
||||
class SignalSpider(scrapy.Spider):
|
||||
name = 'signals'
|
||||
start_urls = ['http://quotes.toscrape.com/page/1/']
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||
return spider
|
||||
|
||||
def item_scraped(self, item):
|
||||
# Send the scraped item to the server
|
||||
d = treq.post(
|
||||
'http://example.com/post',
|
||||
json.dumps(item).encode('ascii'),
|
||||
headers={b'Content-Type': [b'application/json']}
|
||||
)
|
||||
|
||||
# The next item will be scraped only after
|
||||
# deferred (d) is fired
|
||||
return d
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.css('small.author::text').get(),
|
||||
'tags': quote.css('div.tags a.tag::text').getall(),
|
||||
}
|
||||
|
||||
See the :ref:`topics-signals-ref` below to know which signals support
|
||||
:class:`~twisted.internet.defer.Deferred`.
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -66,9 +101,12 @@ Built-in signals reference
|
|||
|
||||
Here's the list of Scrapy built-in signals and their meaning.
|
||||
|
||||
engine_started
|
||||
Engine signals
|
||||
--------------
|
||||
|
||||
engine_started
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: engine_started
|
||||
.. function:: engine_started()
|
||||
|
||||
|
|
@ -81,7 +119,7 @@ engine_started
|
|||
getting fired before :signal:`spider_opened`.
|
||||
|
||||
engine_stopped
|
||||
--------------
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: engine_stopped
|
||||
.. function:: engine_stopped()
|
||||
|
|
@ -91,9 +129,20 @@ engine_stopped
|
|||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
||||
item_scraped
|
||||
Item signals
|
||||
------------
|
||||
|
||||
.. note::
|
||||
As at max :setting:`CONCURRENT_ITEMS` items are processed in
|
||||
parallel, many deferreds are fired together using
|
||||
:class:`~twisted.internet.defer.DeferredList`. Hence the next
|
||||
batch waits for the :class:`~twisted.internet.defer.DeferredList`
|
||||
to fire and then runs the respective item signal handler for
|
||||
the next batch of scraped items.
|
||||
|
||||
item_scraped
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. signal:: item_scraped
|
||||
.. function:: item_scraped(item, response, spider)
|
||||
|
||||
|
|
@ -112,7 +161,7 @@ item_scraped
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
item_dropped
|
||||
------------
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. signal:: item_dropped
|
||||
.. function:: item_dropped(item, response, exception, spider)
|
||||
|
|
@ -137,7 +186,7 @@ item_dropped
|
|||
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
|
||||
|
||||
item_error
|
||||
------------
|
||||
~~~~~~~~~~
|
||||
|
||||
.. signal:: item_error
|
||||
.. function:: item_error(item, response, spider, failure)
|
||||
|
|
@ -159,8 +208,11 @@ item_error
|
|||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
Spider signals
|
||||
--------------
|
||||
|
||||
spider_closed
|
||||
-------------
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: spider_closed
|
||||
.. function:: spider_closed(spider, reason)
|
||||
|
|
@ -183,7 +235,7 @@ spider_closed
|
|||
:type reason: str
|
||||
|
||||
spider_opened
|
||||
-------------
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: spider_opened
|
||||
.. function:: spider_opened(spider)
|
||||
|
|
@ -198,7 +250,7 @@ spider_opened
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
spider_idle
|
||||
-----------
|
||||
~~~~~~~~~~~
|
||||
|
||||
.. signal:: spider_idle
|
||||
.. function:: spider_idle(spider)
|
||||
|
|
@ -228,7 +280,7 @@ spider_idle
|
|||
due to duplication).
|
||||
|
||||
spider_error
|
||||
------------
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. signal:: spider_error
|
||||
.. function:: spider_error(failure, response, spider)
|
||||
|
|
@ -246,8 +298,11 @@ spider_error
|
|||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
Request signals
|
||||
---------------
|
||||
|
||||
request_scheduled
|
||||
-----------------
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: request_scheduled
|
||||
.. function:: request_scheduled(request, spider)
|
||||
|
|
@ -264,7 +319,7 @@ request_scheduled
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
request_dropped
|
||||
---------------
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: request_dropped
|
||||
.. function:: request_dropped(request, spider)
|
||||
|
|
@ -281,7 +336,7 @@ request_dropped
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
request_reached_downloader
|
||||
---------------------------
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: request_reached_downloader
|
||||
.. function:: request_reached_downloader(request, spider)
|
||||
|
|
@ -297,7 +352,7 @@ request_reached_downloader
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
request_left_downloader
|
||||
-----------------------
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: request_left_downloader
|
||||
.. function:: request_left_downloader(request, spider)
|
||||
|
|
@ -315,8 +370,11 @@ request_left_downloader
|
|||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
Response signals
|
||||
----------------
|
||||
|
||||
response_received
|
||||
-----------------
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: response_received
|
||||
.. function:: response_received(response, request, spider)
|
||||
|
|
@ -336,7 +394,7 @@ response_received
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
response_downloaded
|
||||
-------------------
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: response_downloaded
|
||||
.. function:: response_downloaded(response, request, spider)
|
||||
|
|
|
|||
52
pytest.ini
52
pytest.ini
|
|
@ -30,64 +30,56 @@ flake8-ignore =
|
|||
# Issues pending a review:
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128
|
||||
scrapy/commands/fetch.py E401 E128
|
||||
scrapy/commands/fetch.py E128
|
||||
scrapy/commands/genspider.py E128
|
||||
scrapy/commands/parse.py E128
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128
|
||||
scrapy/commands/startproject.py E127 E128
|
||||
scrapy/commands/startproject.py E128
|
||||
scrapy/commands/version.py E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py W504
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E128 E127
|
||||
scrapy/core/scraper.py E128 W504
|
||||
scrapy/core/engine.py E128
|
||||
scrapy/core/scraper.py E128
|
||||
scrapy/core/spidermw.py E126
|
||||
scrapy/core/downloader/contextfactory.py E128 E126
|
||||
scrapy/core/downloader/webclient.py E128 E126
|
||||
scrapy/core/downloader/handlers/ftp.py E128 E127
|
||||
scrapy/core/downloader/handlers/ftp.py E128
|
||||
scrapy/core/downloader/handlers/s3.py E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/httpcache.py E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E128
|
||||
scrapy/downloadermiddlewares/redirect.py W504
|
||||
scrapy/downloadermiddlewares/retry.py E126
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E128 E123
|
||||
scrapy/extensions/closespider.py E128
|
||||
scrapy/extensions/feedexport.py E128
|
||||
scrapy/extensions/httpcache.py E128
|
||||
scrapy/extensions/telnet.py W504
|
||||
# scrapy/http
|
||||
scrapy/http/request/form.py E123
|
||||
scrapy/http/response/__init__.py E128
|
||||
scrapy/http/response/text.py E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E402 W504
|
||||
scrapy/linkextractors/__init__.py E402
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E128
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/files.py E116
|
||||
scrapy/pipelines/media.py E125
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/default_settings.py E114 E116
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/referer.py E129 W504
|
||||
scrapy/spidermiddlewares/referer.py E129
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E402
|
||||
# scrapy/utils
|
||||
scrapy/utils/conf.py E402
|
||||
scrapy/utils/defer.py E128
|
||||
scrapy/utils/deprecate.py E128 E127
|
||||
scrapy/utils/gz.py W504
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/log.py E128
|
||||
scrapy/utils/markup.py F403
|
||||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/request.py E127
|
||||
scrapy/utils/response.py E128
|
||||
scrapy/utils/signal.py E128
|
||||
scrapy/utils/url.py F403 E128 F405
|
||||
|
|
@ -101,10 +93,8 @@ flake8-ignore =
|
|||
scrapy/squeues.py E128
|
||||
# tests
|
||||
tests/__init__.py E402
|
||||
tests/mockserver.py E401 E126 E123
|
||||
tests/mockserver.py E126
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E127
|
||||
tests/test_closespider.py E127
|
||||
tests/test_command_parse.py E128
|
||||
tests/test_command_shell.py E128
|
||||
tests/test_commands.py E128
|
||||
|
|
@ -112,19 +102,18 @@ flake8-ignore =
|
|||
tests/test_crawl.py E741
|
||||
tests/test_crawler.py F841
|
||||
tests/test_dependencies.py F841
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E126 E123
|
||||
tests/test_downloader_handlers.py E124 E128 E126
|
||||
tests/test_downloadermiddleware_cookies.py E741 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_httpcompression.py E126 E123
|
||||
tests/test_downloadermiddleware_httpcompression.py E126
|
||||
tests/test_downloadermiddleware_httpproxy.py E128
|
||||
tests/test_downloadermiddleware_redirect.py E128 E127
|
||||
tests/test_downloadermiddleware_redirect.py E128
|
||||
tests/test_downloadermiddleware_retry.py E128 E126
|
||||
tests/test_dupefilters.py E741 E128 E124
|
||||
tests/test_engine.py E401 E128
|
||||
tests/test_engine.py E128
|
||||
tests/test_exporters.py E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py F841
|
||||
tests/test_http_request.py E402 E127 E128 E128 E126 E123
|
||||
tests/test_http_request.py E402 E128 E128 E126
|
||||
tests/test_http_response.py E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_linkextractors.py E128 E124
|
||||
|
|
@ -136,24 +125,23 @@ flake8-ignore =
|
|||
tests/test_pipeline_images.py F841
|
||||
tests/test_pipeline_media.py E741 E128
|
||||
tests/test_proxy_connect.py E741
|
||||
tests/test_scheduler.py E126 E123
|
||||
tests/test_selector.py E127
|
||||
tests/test_spidermiddleware_httperror.py E128 E127 E121
|
||||
tests/test_scheduler.py E126
|
||||
tests/test_spidermiddleware_httperror.py E128 E121
|
||||
tests/test_spidermiddleware_offsite.py E128 E111
|
||||
tests/test_spidermiddleware_referer.py F841 E125 E124 E121
|
||||
tests/test_spidermiddleware_referer.py F841 E124 E121
|
||||
tests/test_squeues.py E741
|
||||
tests/test_utils_conf.py E128
|
||||
tests/test_utils_datatypes.py E402
|
||||
tests/test_utils_defer.py F841
|
||||
tests/test_utils_deprecate.py F841
|
||||
tests/test_utils_http.py E128 W504
|
||||
tests/test_utils_http.py E128
|
||||
tests/test_utils_iterators.py E128 E129
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_reqser.py E128
|
||||
tests/test_utils_request.py E128
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E124
|
||||
tests/test_utils_url.py E127 E125 E126 E123
|
||||
tests/test_webclient.py E128 E122 E402 E123 E126
|
||||
tests/test_utils_url.py E126
|
||||
tests/test_webclient.py E128 E122 E402 E126
|
||||
tests/test_settings/__init__.py E128
|
||||
tests/test_spiderloader/__init__.py E128
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.0.0
|
||||
2.1.0
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@ class ScrapyCommand:
|
|||
self.settings = None # set in scrapy.cmdline
|
||||
|
||||
def set_crawler(self, crawler):
|
||||
assert not hasattr(self, '_crawler'), "crawler already set"
|
||||
if hasattr(self, '_crawler'):
|
||||
raise RuntimeError("crawler already set")
|
||||
self._crawler = crawler
|
||||
|
||||
def syntax(self):
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@ class ContractsManager:
|
|||
self.contracts[contract.name] = contract
|
||||
|
||||
def tested_methods_from_spidercls(self, spidercls):
|
||||
is_method = re.compile(r"^\s*@", re.MULTILINE).search
|
||||
methods = []
|
||||
for key, value in getmembers(spidercls):
|
||||
if (callable(value) and value.__doc__ and
|
||||
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
|
||||
if callable(value) and value.__doc__ and is_method(value.__doc__):
|
||||
methods.append(key)
|
||||
|
||||
return methods
|
||||
|
|
|
|||
|
|
@ -58,7 +58,11 @@ class ReturnsContract(Contract):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(ReturnsContract, self).__init__(*args, **kwargs)
|
||||
|
||||
assert len(self.args) in [1, 2, 3]
|
||||
if len(self.args) not in [1, 2, 3]:
|
||||
raise ValueError(
|
||||
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
|
||||
% len(self.args)
|
||||
)
|
||||
self.obj_name = self.args[0] or None
|
||||
self.obj_type = self.objects[self.obj_name]
|
||||
|
||||
|
|
|
|||
|
|
@ -94,11 +94,12 @@ class FTPDownloadHandler:
|
|||
def gotClient(self, client, request, filepath):
|
||||
self.client = client
|
||||
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
|
||||
return client.retrieveFile(filepath, protocol)\
|
||||
.addCallbacks(callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,))
|
||||
return client.retrieveFile(filepath, protocol).addCallbacks(
|
||||
callback=self._build_response,
|
||||
callbackArgs=(request, protocol),
|
||||
errback=self._failed,
|
||||
errbackArgs=(request,),
|
||||
)
|
||||
|
||||
def _build_response(self, result, request, protocol):
|
||||
self.result = result
|
||||
|
|
|
|||
|
|
@ -45,8 +45,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def process_response(response):
|
||||
assert response is not None, 'Received None in process_response'
|
||||
if isinstance(response, Request):
|
||||
if response is None:
|
||||
raise TypeError("Received None in process_response")
|
||||
elif isinstance(response, Request):
|
||||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
|
|
|
|||
|
|
@ -73,7 +73,8 @@ class ExecutionEngine:
|
|||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
"""Start the execution engine"""
|
||||
assert not self.running, "Engine already running"
|
||||
if self.running:
|
||||
raise RuntimeError("Engine already running")
|
||||
self.start_time = time()
|
||||
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
|
||||
self.running = True
|
||||
|
|
@ -82,7 +83,8 @@ class ExecutionEngine:
|
|||
|
||||
def stop(self):
|
||||
"""Stop the execution engine gracefully"""
|
||||
assert self.running, "Engine not running"
|
||||
if not self.running:
|
||||
raise RuntimeError("Engine not running")
|
||||
self.running = False
|
||||
dfd = self._close_all_spiders()
|
||||
return dfd.addBoth(lambda _: self._finish_stopping_engine())
|
||||
|
|
@ -165,7 +167,11 @@ class ExecutionEngine:
|
|||
return d
|
||||
|
||||
def _handle_downloader_output(self, response, request, spider):
|
||||
assert isinstance(response, (Request, Response, Failure)), response
|
||||
if not isinstance(response, (Request, Response, Failure)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Request, Response or Failure, got %s: %r"
|
||||
% (type(response), response)
|
||||
)
|
||||
# downloader middleware can return requests (for example, redirects)
|
||||
if isinstance(response, Request):
|
||||
self.crawl(response, spider)
|
||||
|
|
@ -205,8 +211,8 @@ class ExecutionEngine:
|
|||
return not bool(self.slot)
|
||||
|
||||
def crawl(self, request, spider):
|
||||
assert spider in self.open_spiders, \
|
||||
"Spider %r not opened when crawling: %s" % (spider.name, request)
|
||||
if spider not in self.open_spiders:
|
||||
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
|
||||
self.schedule(request, spider)
|
||||
self.slot.nextcall.schedule()
|
||||
|
||||
|
|
@ -224,15 +230,18 @@ class ExecutionEngine:
|
|||
|
||||
def _downloaded(self, response, slot, request, spider):
|
||||
slot.remove_request(request)
|
||||
return self.download(response, spider) \
|
||||
if isinstance(response, Request) else response
|
||||
return self.download(response, spider) if isinstance(response, Request) else response
|
||||
|
||||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
slot.add_request(request)
|
||||
|
||||
def _on_success(response):
|
||||
assert isinstance(response, (Response, Request))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Response or Request, got %s: %r"
|
||||
% (type(response), response)
|
||||
)
|
||||
if isinstance(response, Response):
|
||||
response.request = request # tie request to response received
|
||||
logkws = self.logformatter.crawled(request, response, spider)
|
||||
|
|
@ -253,8 +262,8 @@ class ExecutionEngine:
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def open_spider(self, spider, start_requests=(), close_if_idle=True):
|
||||
assert self.has_capacity(), "No free spider slot when opening %r" % \
|
||||
spider.name
|
||||
if not self.has_capacity():
|
||||
raise RuntimeError("No free spider slot when opening %r" % spider.name)
|
||||
logger.info("Spider opened", extra={'spider': spider})
|
||||
nextcall = CallLaterOnce(self._next_request, spider)
|
||||
scheduler = self.scheduler_cls.from_crawler(self.crawler)
|
||||
|
|
|
|||
|
|
@ -123,7 +123,11 @@ class Scraper:
|
|||
def _scrape(self, response, request, spider):
|
||||
"""Handle the downloaded response or failure through the spider
|
||||
callback/errback"""
|
||||
assert isinstance(response, (Response, Failure))
|
||||
if not isinstance(response, (Response, Failure)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Response or Failure, got %s: %r"
|
||||
% (type(response), response)
|
||||
)
|
||||
|
||||
dfd = self._scrape2(response, request, spider) # returns spider's processed output
|
||||
dfd.addErrback(self.handle_spider_error, request, response, spider)
|
||||
|
|
|
|||
|
|
@ -78,7 +78,8 @@ class Crawler:
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def crawl(self, *args, **kwargs):
|
||||
assert not self.crawling, "Crawling already taking place"
|
||||
if self.crawling:
|
||||
raise RuntimeError("Crawling already taking place")
|
||||
self.crawling = True
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if (request.meta.get('dont_redirect', False) or
|
||||
response.status in getattr(spider, 'handle_httpstatus_list', []) or
|
||||
response.status in request.meta.get('handle_httpstatus_list', []) or
|
||||
request.meta.get('handle_httpstatus_all', False)):
|
||||
if (
|
||||
request.meta.get('dont_redirect', False)
|
||||
or response.status in getattr(spider, 'handle_httpstatus_list', [])
|
||||
or response.status in request.meta.get('handle_httpstatus_list', [])
|
||||
or request.meta.get('handle_httpstatus_all', False)
|
||||
):
|
||||
return response
|
||||
|
||||
allowed_status = (301, 302, 303, 307, 308)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class CloseSpider:
|
|||
'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
|
||||
'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'),
|
||||
'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
|
||||
}
|
||||
}
|
||||
|
||||
if not any(self.close_on.values()):
|
||||
raise NotConfigured
|
||||
|
|
|
|||
|
|
@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
|
|||
"""An implementation of IPortal"""
|
||||
@defers
|
||||
def login(self_, credentials, mind, *interfaces):
|
||||
if not (credentials.username == self.username.encode('utf8') and
|
||||
credentials.checkPassword(self.password.encode('utf8'))):
|
||||
if not (
|
||||
credentials.username == self.username.encode('utf8')
|
||||
and credentials.checkPassword(self.password.encode('utf8'))
|
||||
):
|
||||
raise ValueError("Invalid credentials")
|
||||
|
||||
protocol = telnet.TelnetBootstrapProtocol(
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ class Request(object_ref):
|
|||
self.method = str(method).upper()
|
||||
self._set_url(url)
|
||||
self._set_body(body)
|
||||
assert isinstance(priority, int), "Request priority not an integer: %r" % priority
|
||||
if not isinstance(priority, int):
|
||||
raise TypeError("Request priority not an integer: %r" % priority)
|
||||
self.priority = priority
|
||||
|
||||
if callback is not None and not callable(callback):
|
||||
|
|
|
|||
|
|
@ -178,12 +178,11 @@ def _get_clickable(clickdata, form):
|
|||
if the latter is given. If not, it returns the first
|
||||
clickable element found
|
||||
"""
|
||||
clickables = [
|
||||
el for el in form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"})
|
||||
]
|
||||
clickables = list(form.xpath(
|
||||
'descendant::input[re:test(@type, "^(submit|image)$", "i")]'
|
||||
'|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]',
|
||||
namespaces={"re": "http://exslt.org/regular-expressions"}
|
||||
))
|
||||
if not clickables:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -61,8 +61,7 @@ class FilteringLinkExtractor:
|
|||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
|
||||
if (issubclass(cls, FilteringLinkExtractor) and
|
||||
not issubclass(cls, LxmlLinkExtractor)):
|
||||
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
|
||||
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
|
||||
'please use scrapy.linkextractors.LinkExtractor instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
|
|||
|
|
@ -106,7 +106,8 @@ class S3FilesStore:
|
|||
else:
|
||||
from boto.s3.connection import S3Connection
|
||||
self.S3Connection = S3Connection
|
||||
assert uri.startswith('s3://')
|
||||
if not uri.startswith("s3://"):
|
||||
raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri)
|
||||
self.bucket, self.prefix = uri[5:].split('/', 1)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
|
|
@ -229,6 +230,20 @@ class GCSFilesStore:
|
|||
bucket, prefix = uri[5:].split('/', 1)
|
||||
self.bucket = client.bucket(bucket)
|
||||
self.prefix = prefix
|
||||
permissions = self.bucket.test_iam_permissions(
|
||||
['storage.objects.get', 'storage.objects.create']
|
||||
)
|
||||
if 'storage.objects.get' not in permissions:
|
||||
logger.warning(
|
||||
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
if 'storage.objects.create' not in permissions:
|
||||
logger.error(
|
||||
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||
{'bucket': bucket}
|
||||
)
|
||||
|
||||
def stat_file(self, path, info):
|
||||
def _onsuccess(blob):
|
||||
|
|
@ -266,7 +281,8 @@ class FTPFilesStore:
|
|||
USE_ACTIVE_MODE = None
|
||||
|
||||
def __init__(self, uri):
|
||||
assert uri.startswith('ftp://')
|
||||
if not uri.startswith("ftp://"):
|
||||
raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri)
|
||||
u = urlparse(uri)
|
||||
self.port = u.port
|
||||
self.host = u.hostname
|
||||
|
|
|
|||
|
|
@ -43,8 +43,7 @@ class MediaPipeline:
|
|||
if allow_redirects:
|
||||
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
|
||||
|
||||
def _key_for_pipe(self, key, base_class_name=None,
|
||||
settings=None):
|
||||
def _key_for_pipe(self, key, base_class_name=None, settings=None):
|
||||
"""
|
||||
>>> MediaPipeline()._key_for_pipe("IMAGES")
|
||||
'IMAGES'
|
||||
|
|
@ -55,8 +54,11 @@ class MediaPipeline:
|
|||
"""
|
||||
class_name = self.__class__.__name__
|
||||
formatted_key = "{}_{}".format(class_name.upper(), key)
|
||||
if class_name == base_class_name or not base_class_name \
|
||||
or (settings and not settings.get(formatted_key)):
|
||||
if (
|
||||
not base_class_name
|
||||
or class_name == base_class_name
|
||||
or settings and not settings.get(formatted_key)
|
||||
):
|
||||
return key
|
||||
return formatted_key
|
||||
|
||||
|
|
|
|||
|
|
@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
|
|||
name = POLICY_STRICT_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
if (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
|
|||
origin = self.origin(response_url)
|
||||
if origin == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
elif ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
elif (
|
||||
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
|
||||
or not self.tls_protected(response_url)
|
||||
):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
|
|||
stacklevel=3)
|
||||
|
||||
|
||||
def create_deprecated_class(name, new_class, clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from "
|
||||
"deprecated class {old}, please inherit "
|
||||
"from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, "
|
||||
"instantiate {new} instead."):
|
||||
def create_deprecated_class(
|
||||
name,
|
||||
new_class,
|
||||
clsdict=None,
|
||||
warn_category=ScrapyDeprecationWarning,
|
||||
warn_once=True,
|
||||
old_class_path=None,
|
||||
new_class_path=None,
|
||||
subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
|
||||
instance_warn_message="{cls} is deprecated, instantiate {new} instead."
|
||||
):
|
||||
"""
|
||||
Return a "deprecated" class that causes its subclasses to issue a warning.
|
||||
Subclasses of ``new_class`` are considered subclasses of this class.
|
||||
|
|
|
|||
|
|
@ -52,8 +52,7 @@ def is_gzipped(response):
|
|||
"""Return True if the response is gzipped, or False otherwise"""
|
||||
ctype = response.headers.get('Content-Type', b'')
|
||||
cenc = response.headers.get('Content-Encoding', b'').lower()
|
||||
return (_is_gzipped(ctype) or
|
||||
(_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
|
||||
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
|
||||
|
||||
|
||||
def gzip_magic_number(response):
|
||||
|
|
|
|||
|
|
@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
|
|||
|
||||
def _body_or_str(obj, unicode=True):
|
||||
expected_types = (Response, str, bytes)
|
||||
assert isinstance(obj, expected_types), \
|
||||
"obj must be %s, not %s" % (
|
||||
" or ".join(t.__name__ for t in expected_types),
|
||||
type(obj).__name__)
|
||||
if not isinstance(obj, expected_types):
|
||||
expected_types_str = " or ".join(t.__name__ for t in expected_types)
|
||||
raise TypeError(
|
||||
"Object %r must be %s, not %s"
|
||||
% (obj, expected_types_str, type(obj).__name__)
|
||||
)
|
||||
if isinstance(obj, Response):
|
||||
if not unicode:
|
||||
return obj.body
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object
|
|||
def listen_tcp(portrange, host, factory):
|
||||
"""Like reactor.listenTCP but tries different ports in a range."""
|
||||
from twisted.internet import reactor
|
||||
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
|
||||
if len(portrange) > 2:
|
||||
raise ValueError("invalid portrange: %s" % portrange)
|
||||
if not portrange:
|
||||
return reactor.listenTCP(0, factory, interface=host)
|
||||
if not hasattr(portrange, '__iter__'):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
"""
|
||||
Helper functions for serializing (and deserializing) requests.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.misc import load_object
|
||||
|
|
@ -68,20 +70,6 @@ def request_from_dict(d, spider=None):
|
|||
)
|
||||
|
||||
|
||||
def _is_private_method(name):
|
||||
return name.startswith('__') and not name.endswith('__')
|
||||
|
||||
|
||||
def _mangle_private_name(obj, func, name):
|
||||
qualname = getattr(func, '__qualname__', None)
|
||||
if qualname is None:
|
||||
classname = obj.__class__.__name__.lstrip('_')
|
||||
return '_%s%s' % (classname, name)
|
||||
else:
|
||||
splits = qualname.split('.')
|
||||
return '_%s%s' % (splits[-2], splits[-1])
|
||||
|
||||
|
||||
def _find_method(obj, func):
|
||||
if obj:
|
||||
try:
|
||||
|
|
@ -90,10 +78,17 @@ def _find_method(obj, func):
|
|||
pass
|
||||
else:
|
||||
if func_self is obj:
|
||||
name = func.__func__.__name__
|
||||
if _is_private_method(name):
|
||||
return _mangle_private_name(obj, func, name)
|
||||
return name
|
||||
members = inspect.getmembers(obj, predicate=inspect.ismethod)
|
||||
for name, obj_func in members:
|
||||
# We need to use __func__ to access the original
|
||||
# function object because instance method objects
|
||||
# are generated each time attribute is retrieved from
|
||||
# instance.
|
||||
#
|
||||
# Reference: The standard type hierarchy
|
||||
# https://docs.python.org/3/reference/datamodel.html
|
||||
if obj_func.__func__ is func.__func__:
|
||||
return name
|
||||
raise ValueError("Function %s is not a method of: %s" % (func, obj))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
|||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
|
|||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
||||
assert self.seedsseen, \
|
||||
'All start requests consumed before any download happened'
|
||||
assert self.seedsseen, 'All start requests consumed before any download happened'
|
||||
|
||||
def parse(self, response):
|
||||
self.seedsseen.append(response.meta.get('seed'))
|
||||
|
|
|
|||
|
|
@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
|
|||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_errorcount')
|
||||
key = 'spider_exceptions/{name}'\
|
||||
.format(name=crawler.spider.exception_cls.__name__)
|
||||
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
|
||||
errorcount = crawler.stats.get_value(key)
|
||||
self.assertTrue(errorcount >= close_on)
|
||||
|
||||
|
|
|
|||
|
|
@ -828,11 +828,15 @@ class S3TestCase(unittest.TestCase):
|
|||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 21:15:45 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg',
|
||||
method='PUT',
|
||||
headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
},
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
|
|
@ -912,11 +916,10 @@ class S3TestCase(unittest.TestCase):
|
|||
# ensure that spaces are quoted properly before signing
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request(
|
||||
("s3://johnsmith/photos/my puppy.jpg"
|
||||
"?response-content-disposition=my puppy.jpg"),
|
||||
"s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg",
|
||||
method='GET',
|
||||
headers={'Date': date},
|
||||
)
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(
|
||||
|
|
@ -1096,8 +1099,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype_encoding(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, 'A brief note')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "US-ASCII")
|
||||
|
||||
request = Request("data:,A%20brief%20note")
|
||||
|
|
@ -1106,8 +1108,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_default_mediatype(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
request = Request("data:;charset=iso-8859-7,%be%d3%be")
|
||||
|
|
@ -1125,8 +1126,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
def test_mediatype_parameters(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response),
|
||||
responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
|
||||
request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
|
|||
for fmt in self.test_formats:
|
||||
rsp = self.test_responses[fmt]
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert isinstance(new, XmlResponse), \
|
||||
'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
|
||||
assert isinstance(new, XmlResponse), error_msg
|
||||
assert_samelines(self, new.body, self.uncompressed_body, fmt)
|
||||
|
||||
def test_plain_response(self):
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding
|
|||
SAMPLEDIR = join(tests_datadir, 'compressed')
|
||||
|
||||
FORMAT = {
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br')
|
||||
}
|
||||
'gzip': ('html-gzip.bin', 'gzip'),
|
||||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br'),
|
||||
}
|
||||
|
||||
|
||||
class HttpCompressionTest(TestCase):
|
||||
|
|
@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase):
|
|||
body = sample.read()
|
||||
|
||||
headers = {
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
'Server': 'Yaws/1.49 Yet Another Web Server',
|
||||
'Date': 'Sun, 08 Mar 2009 00:41:03 GMT',
|
||||
'Content-Length': len(body),
|
||||
'Content-Type': 'text/html',
|
||||
'Content-Encoding': contentencoding,
|
||||
}
|
||||
|
||||
response = Response('http://scrapytest.org/', body=body, headers=headers)
|
||||
response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
|
||||
|
|
|
|||
|
|
@ -184,8 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertIs(r, rsp)
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list':
|
||||
[404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
|
||||
_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
|
||||
|
||||
def test_latin1_location(self):
|
||||
|
|
|
|||
|
|
@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_bytes(self):
|
||||
data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'latin1')
|
||||
self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
|
||||
|
|
@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
data = {'price': u'£ 100'}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data,
|
||||
encoding='latin1')
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, b'price=%A3+100')
|
||||
|
||||
|
|
@ -469,7 +467,7 @@ class FormRequestTest(RequestTest):
|
|||
</form>""",
|
||||
url="http://www.example.com/this/list.html",
|
||||
encoding='latin1',
|
||||
)
|
||||
)
|
||||
req = self.request_class.from_response(response,
|
||||
formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ class MockCrawler(Crawler):
|
|||
def __init__(self, priority_queue_cls, jobdir):
|
||||
|
||||
settings = dict(
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
|
||||
)
|
||||
SCHEDULER_DEBUG=False,
|
||||
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
|
||||
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
|
||||
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
|
||||
JOBDIR=jobdir,
|
||||
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
|
||||
)
|
||||
super(MockCrawler, self).__init__(Spider, settings)
|
||||
self.engine = MockEngine(downloader=MockDownloader())
|
||||
|
||||
|
|
@ -305,10 +305,12 @@ class StartUrlsSpider(Spider):
|
|||
class TestIntegrationWithDownloaderAwareInMemory(TestCase):
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler(
|
||||
StartUrlsSpider,
|
||||
{'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
|
||||
)
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
|
||||
},
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
|
|
@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase):
|
|||
|
||||
def _incompatible(self):
|
||||
settings = dict(
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1
|
||||
)
|
||||
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
|
||||
CONCURRENT_REQUESTS_PER_IP=1,
|
||||
)
|
||||
crawler = Crawler(Spider, settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = Spider(name='spider')
|
||||
|
|
|
|||
|
|
@ -75,8 +75,7 @@ class SelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
|
||||
[u'\xa3'])
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
|
|
|
|||
|
|
@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
|
|||
self.mw.process_spider_input(self.res402, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
|
|||
self.mw.process_spider_input(self.res404, self.spider))
|
||||
|
||||
def test_meta_overrides_settings(self):
|
||||
request = Request('http://scrapytest.org',
|
||||
meta={'handle_httpstatus_list': [404]})
|
||||
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
|
||||
res404 = self.res404.copy()
|
||||
res404.request = request
|
||||
res402 = self.res402.copy()
|
||||
|
|
|
|||
|
|
@ -514,32 +514,32 @@ class TestSettingsPolicyByName(TestCase):
|
|||
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEqual(mw.default_policy, p)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
|
|||
chunked_body += "8\r\n" + "sequence\r\n"
|
||||
chunked_body += "0\r\n\r\n"
|
||||
body = decode_chunked_transfer(chunked_body)
|
||||
self.assertEqual(body,
|
||||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
self.assertEqual(
|
||||
body,
|
||||
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
def test_xmliter_objtype_exception(self):
|
||||
i = self.xmliter(42, 'product')
|
||||
self.assertRaises(AssertionError, next, i)
|
||||
self.assertRaises(TypeError, next, i)
|
||||
|
||||
def test_xmliter_encoding(self):
|
||||
body = (
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import unittest
|
|||
|
||||
from scrapy.http import Request, FormRequest
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
|
||||
from scrapy.utils.reqser import request_to_dict, request_from_dict
|
||||
|
||||
|
||||
class RequestSerializationTest(unittest.TestCase):
|
||||
|
|
@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
errback=self.spider.handle_error)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_reference_callback_serialization(self):
|
||||
r = Request("http://www.example.com",
|
||||
callback=self.spider.parse_item_reference,
|
||||
errback=self.spider.handle_error_reference)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
request_dict = request_to_dict(r, self.spider)
|
||||
self.assertEqual(request_dict['callback'], 'parse_item_reference')
|
||||
self.assertEqual(request_dict['errback'], 'handle_error_reference')
|
||||
|
||||
def test_private_reference_callback_serialization(self):
|
||||
r = Request("http://www.example.com",
|
||||
callback=self.spider._TestSpider__parse_item_reference,
|
||||
errback=self.spider._TestSpider__handle_error_reference)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
request_dict = request_to_dict(r, self.spider)
|
||||
self.assertEqual(request_dict['callback'],
|
||||
'_TestSpider__parse_item_reference')
|
||||
self.assertEqual(request_dict['errback'],
|
||||
'_TestSpider__handle_error_reference')
|
||||
|
||||
def test_private_callback_serialization(self):
|
||||
r = Request("http://www.example.com",
|
||||
callback=self.spider._TestSpider__parse_item_private,
|
||||
|
|
@ -81,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
errback=self.spider.handle_error)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_private_callback_name_matching(self):
|
||||
self.assertTrue(_is_private_method('__a'))
|
||||
self.assertTrue(_is_private_method('__a_'))
|
||||
self.assertTrue(_is_private_method('__a_a'))
|
||||
self.assertTrue(_is_private_method('__a_a_'))
|
||||
self.assertTrue(_is_private_method('__a__a'))
|
||||
self.assertTrue(_is_private_method('__a__a_'))
|
||||
self.assertTrue(_is_private_method('__a___a'))
|
||||
self.assertTrue(_is_private_method('__a___a_'))
|
||||
self.assertTrue(_is_private_method('___a'))
|
||||
self.assertTrue(_is_private_method('___a_'))
|
||||
self.assertTrue(_is_private_method('___a_a'))
|
||||
self.assertTrue(_is_private_method('___a_a_'))
|
||||
self.assertTrue(_is_private_method('____a_a_'))
|
||||
|
||||
self.assertFalse(_is_private_method('_a'))
|
||||
self.assertFalse(_is_private_method('_a_'))
|
||||
self.assertFalse(_is_private_method('__a__'))
|
||||
self.assertFalse(_is_private_method('__'))
|
||||
self.assertFalse(_is_private_method('___'))
|
||||
self.assertFalse(_is_private_method('____'))
|
||||
|
||||
def _assert_mangles_to(self, obj, name):
|
||||
func = getattr(obj, name)
|
||||
self.assertEqual(
|
||||
_mangle_private_name(obj, func, func.__name__),
|
||||
name
|
||||
)
|
||||
|
||||
def test_private_name_mangling(self):
|
||||
self._assert_mangles_to(
|
||||
self.spider, '_TestSpider__parse_item_private')
|
||||
self._assert_mangles_to(
|
||||
self.spider, '_TestSpiderMixin__mixin_callback')
|
||||
|
||||
def test_unserializable_callback1(self):
|
||||
r = Request("http://www.example.com", callback=lambda x: x)
|
||||
self.assertRaises(ValueError, request_to_dict, r)
|
||||
|
|
@ -125,14 +110,49 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
r = Request("http://www.example.com", callback=self.spider.parse_item)
|
||||
self.assertRaises(ValueError, request_to_dict, r)
|
||||
|
||||
def test_unserializable_callback3(self):
|
||||
"""Parser method is removed or replaced dynamically."""
|
||||
|
||||
class MySpider(Spider):
|
||||
|
||||
name = 'my_spider'
|
||||
|
||||
def parse(self, response):
|
||||
pass
|
||||
|
||||
spider = MySpider()
|
||||
r = Request("http://www.example.com", callback=spider.parse)
|
||||
setattr(spider, 'parse', None)
|
||||
self.assertRaises(ValueError, request_to_dict, r, spider=spider)
|
||||
|
||||
|
||||
class TestSpiderMixin:
|
||||
def __mixin_callback(self, response):
|
||||
pass
|
||||
|
||||
|
||||
def parse_item(response):
|
||||
pass
|
||||
|
||||
|
||||
def handle_error(failure):
|
||||
pass
|
||||
|
||||
|
||||
def private_parse_item(response):
|
||||
pass
|
||||
|
||||
|
||||
def private_handle_error(failure):
|
||||
pass
|
||||
|
||||
|
||||
class TestSpider(Spider, TestSpiderMixin):
|
||||
name = 'test'
|
||||
parse_item_reference = parse_item
|
||||
handle_error_reference = handle_error
|
||||
__parse_item_reference = private_parse_item
|
||||
__handle_error_reference = private_handle_error
|
||||
|
||||
def parse_item(self, response):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -80,114 +80,124 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
class AddHttpIfNoScheme(unittest.TestCase):
|
||||
|
||||
def test_add_scheme(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
|
||||
|
||||
def test_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
'http://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//example.com'),
|
||||
'http://example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//example.com'), 'http://example.com')
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'
|
||||
)
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
self.assertEqual(
|
||||
add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
|
||||
'ftp://www.example.com')
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
|
||||
|
||||
|
||||
class GuessSchemeTest(unittest.TestCase):
|
||||
|
|
@ -211,41 +221,49 @@ def create_skipped_scheme_t(args):
|
|||
return do_expected
|
||||
|
||||
|
||||
for k, args in enumerate([
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
for k, args in enumerate(
|
||||
[
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
('../index.html', 'file://'),
|
||||
('../../index.html', 'file://'),
|
||||
('./data/index.html', 'file://'),
|
||||
('.hidden/data/index.html', 'file://'),
|
||||
('/home/user/www/index.html', 'file://'),
|
||||
('//home/user/www/index.html', 'file://'),
|
||||
('file:///home/user/www/index.html', 'file://'),
|
||||
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
('index.html', 'http://'),
|
||||
('example.com', 'http://'),
|
||||
('www.example.com', 'http://'),
|
||||
('www.example.com/index.html', 'http://'),
|
||||
('http://example.com', 'http://'),
|
||||
('http://example.com/index.html', 'http://'),
|
||||
('localhost', 'http://'),
|
||||
('localhost/index.html', 'http://'),
|
||||
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
|
||||
], start=1):
|
||||
# some corner cases (default to http://)
|
||||
('/', 'http://'),
|
||||
('.../test', 'http://'),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, args in enumerate([
|
||||
(r'C:\absolute\path\to\a\file.html', 'file://',
|
||||
'Windows filepath are not supported for scrapy shell'),
|
||||
], start=1):
|
||||
for k, args in enumerate(
|
||||
[
|
||||
(
|
||||
r'C:\absolute\path\to\a\file.html',
|
||||
'file://',
|
||||
'Windows filepath are not supported for scrapy shell',
|
||||
),
|
||||
],
|
||||
start=1,
|
||||
):
|
||||
t_method = create_skipped_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
|
@ -281,7 +299,7 @@ class StripUrl(unittest.TestCase):
|
|||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
|
||||
def test_credentials(self):
|
||||
|
|
@ -294,7 +312,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
|
|
@ -313,7 +331,7 @@ class StripUrl(unittest.TestCase):
|
|||
# password: "user@domain.com"
|
||||
('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
|
|
@ -341,7 +359,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
|
|
@ -369,7 +387,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
|
|
@ -397,7 +415,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
|
|
@ -413,7 +431,7 @@ class StripUrl(unittest.TestCase):
|
|||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
},
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})))
|
||||
}),
|
||||
))
|
||||
|
||||
self._test(factory,
|
||||
b"GET /bar HTTP/1.0\r\n"
|
||||
|
|
|
|||
12
tox.ini
12
tox.ini
|
|
@ -102,16 +102,6 @@ setenv = {[docs]setenv}
|
|||
commands =
|
||||
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
|
||||
|
||||
[asyncio]
|
||||
[testenv:asyncio]
|
||||
commands =
|
||||
{[testenv]commands} --reactor=asyncio
|
||||
|
||||
[testenv:py35-asyncio]
|
||||
basepython = python3.5
|
||||
deps = {[testenv]deps}
|
||||
commands = {[asyncio]commands}
|
||||
|
||||
[testenv:py38-asyncio]
|
||||
basepython = python3.8
|
||||
deps = {[testenv]deps}
|
||||
commands = {[asyncio]commands}
|
||||
|
|
|
|||
Loading…
Reference in New Issue