mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into user-friendlier-tox
This commit is contained in:
Adrián Chaves
GitHub
commit
d7b1c138f1
10
conftest.py
10
conftest.py
|
|
@ -1,4 +1,3 @@
|
|||
import six
|
||||
import pytest
|
||||
|
||||
|
||||
|
|
@ -8,11 +7,10 @@ collect_ignore = [
|
|||
]
|
||||
|
||||
|
||||
if six.PY3:
|
||||
for line in open('tests/py3-ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
for line in open('tests/ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
|
|||
13
docs/conf.py
13
docs/conf.py
|
|
@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
|
|||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = [
|
||||
'hoverxref.extension',
|
||||
'notfound.extension',
|
||||
'scrapydocs',
|
||||
'sphinx.ext.autodoc',
|
||||
|
|
@ -274,6 +275,16 @@ coverage_ignore_pyobjects = [
|
|||
# -------------------------------------
|
||||
|
||||
intersphinx_mapping = {
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/stable', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
|
||||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
}
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
# ------------------------------------
|
||||
|
||||
hoverxref_auto_ref = True
|
||||
|
|
|
|||
|
|
@ -194,8 +194,9 @@ documentation instead of duplicating the docstring in files within the
|
|||
Tests
|
||||
=====
|
||||
|
||||
Tests are implemented using the `Twisted unit-testing framework`_, running
|
||||
tests requires `tox`_.
|
||||
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||
<twisted:core/development/policy/test-standard>`. Running tests requires
|
||||
:doc:`tox <tox:index>`.
|
||||
|
||||
.. _running-tests:
|
||||
|
||||
|
|
@ -210,37 +211,37 @@ To run a specific test (say ``tests/test_loader.py``) use:
|
|||
|
||||
``tox -- tests/test_loader.py``
|
||||
|
||||
To run the tests on a specific tox_ environment, use ``-e <name>`` with an
|
||||
environment name from ``tox.ini``. For example, to run the tests with Python
|
||||
3.6 use::
|
||||
To run the tests on a specific :doc:`tox <tox:index>` environment, use
|
||||
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
|
||||
the tests with Python 3.6 use::
|
||||
|
||||
tox -e py36
|
||||
|
||||
You can also specify a comma-separated list of environmets, and use `tox’s
|
||||
parallel mode`_ to run the tests on multiple environments in parallel::
|
||||
You can also specify a comma-separated list of environmets, and use :ref:`tox’s
|
||||
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
|
||||
parallel::
|
||||
|
||||
tox -e py37,py38 -p auto
|
||||
|
||||
To pass command-line options to pytest_, add them after ``--`` in your call to
|
||||
tox_. Using ``--`` overrides the default positional arguments defined in
|
||||
``tox.ini``, so you must include those default positional arguments
|
||||
(``scrapy tests``) after ``--`` as well::
|
||||
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
|
||||
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
|
||||
default positional arguments defined in ``tox.ini``, so you must include those
|
||||
default positional arguments (``scrapy tests``) after ``--`` as well::
|
||||
|
||||
tox -- scrapy tests -x # stop after first failure
|
||||
|
||||
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
|
||||
the Python 3.6 tox_ environment using all your CPU cores::
|
||||
the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
|
||||
|
||||
tox -e py36 -- scrapy tests -n auto
|
||||
|
||||
To see coverage report install `coverage`_ (``pip install coverage``) and run:
|
||||
To see coverage report install :doc:`coverage <coverage:index>`
|
||||
(``pip install coverage``) and run:
|
||||
|
||||
``coverage report``
|
||||
|
||||
see output of ``coverage --help`` for more options like html or xml report.
|
||||
|
||||
.. _coverage: https://pypi.python.org/pypi/coverage
|
||||
|
||||
Writing tests
|
||||
-------------
|
||||
|
||||
|
|
@ -261,13 +262,9 @@ And their unit-tests are in::
|
|||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: https://reddit.com/r/scrapy
|
||||
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
|
||||
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
|
||||
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
|
||||
.. _open issues: https://github.com/scrapy/scrapy/issues
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _pytest: https://docs.pytest.org/en/latest/usage.html
|
||||
.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
|
||||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.xpath('span/small/text()').get(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr("href")').get()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
-r ../requirements-py3.txt
|
||||
Sphinx>=2.1
|
||||
sphinx-hoverxref
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
|
|
|
|||
|
|
@ -273,5 +273,3 @@ class (which they all inherit from).
|
|||
|
||||
Close the given spider. After this is called, no more specific stats
|
||||
can be accessed or collected.
|
||||
|
||||
.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
|
|
|
|||
|
|
@ -166,11 +166,10 @@ for concurrency.
|
|||
For more information about asynchronous programming and Twisted see these
|
||||
links:
|
||||
|
||||
* `Introduction to Deferreds in Twisted`_
|
||||
* :doc:`twisted:core/howto/defer-intro`
|
||||
* `Twisted - hello, asynchronous programming`_
|
||||
* `Twisted Introduction - Krondo`_
|
||||
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. highlight:: none
|
||||
|
||||
.. _topics-commands:
|
||||
|
||||
=================
|
||||
|
|
@ -66,7 +68,9 @@ structure by default, similar to this::
|
|||
|
||||
The directory where the ``scrapy.cfg`` file resides is known as the *project
|
||||
root directory*. That file contains the name of the python module that defines
|
||||
the project settings. Here is an example::
|
||||
the project settings. Here is an example:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject.settings
|
||||
|
|
@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
|
|||
shared by multiple Scrapy projects, each with its own settings module.
|
||||
|
||||
In that case, you must define one or more aliases for those settings modules
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file::
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[settings]
|
||||
default = myproject1.settings
|
||||
|
|
@ -277,6 +283,8 @@ check
|
|||
|
||||
Run contract checks.
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy check -l
|
||||
|
|
@ -294,6 +302,8 @@ Usage examples::
|
|||
[FAILED] first_spider:parse
|
||||
>>> Returned 92 requests, expected 0..4
|
||||
|
||||
.. skip: end
|
||||
|
||||
.. command:: list
|
||||
|
||||
list
|
||||
|
|
@ -481,6 +491,8 @@ Supported options:
|
|||
|
||||
* ``--verbose`` or ``-v``: display information for each depth level
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy parse http://www.example.com/ -c parse_item
|
||||
|
|
@ -495,6 +507,8 @@ Usage example::
|
|||
# Requests -----------------------------------------------------------------
|
||||
[]
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
.. command:: settings
|
||||
|
||||
|
|
@ -573,7 +587,9 @@ Default: ``''`` (empty string)
|
|||
A module to use for looking up custom Scrapy commands. This is used to add custom
|
||||
commands for your Scrapy project.
|
||||
|
||||
Example::
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
COMMANDS_MODULE = 'mybot.commands'
|
||||
|
||||
|
|
@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
|
|||
``scrapy.commands`` section in the entry points of the library ``setup.py``
|
||||
file.
|
||||
|
||||
The following example adds ``my_command`` command::
|
||||
The following example adds ``my_command`` command:
|
||||
|
||||
.. skip: next
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
|
|||
of the spider at the method level. It has the advantage of being flexible and
|
||||
simple to use, but does not allow debugging code inside a method.
|
||||
|
||||
.. highlight:: none
|
||||
|
||||
.. skip: start
|
||||
|
||||
In order to see the item scraped from a specific url::
|
||||
|
||||
$ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
|
||||
|
|
@ -85,6 +89,8 @@ using::
|
|||
|
||||
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
|
||||
|
||||
.. skip: end
|
||||
|
||||
|
||||
Scrapy Shell
|
||||
============
|
||||
|
|
@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides
|
|||
showing the response received and the output. How to debug the situation when
|
||||
``parse_details`` sometimes receives no item?
|
||||
|
||||
.. highlight:: python
|
||||
|
||||
Fortunately, the :command:`shell` is your bread and butter in this case (see
|
||||
:ref:`topics-shell-inspect-response`)::
|
||||
|
||||
|
|
|
|||
|
|
@ -9,13 +9,13 @@ Sending e-mail
|
|||
|
||||
Although Python makes sending e-mails relatively easy via the `smtplib`_
|
||||
library, Scrapy provides its own facility for sending e-mails which is very
|
||||
easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
|
||||
interfering with the non-blocking IO of the crawler. It also provides a
|
||||
simple API for sending attachments and it's very easy to configure, with a few
|
||||
:ref:`settings <topics-email-settings>`.
|
||||
easy to use and it's implemented using :doc:`Twisted non-blocking IO
|
||||
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
|
||||
IO of the crawler. It also provides a simple API for sending attachments and
|
||||
it's very easy to configure, with a few :ref:`settings
|
||||
<topics-email-settings>`.
|
||||
|
||||
.. _smtplib: https://docs.python.org/2/library/smtplib.html
|
||||
.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
|
||||
|
||||
Quick example
|
||||
=============
|
||||
|
|
@ -39,7 +39,8 @@ MailSender class reference
|
|||
==========================
|
||||
|
||||
MailSender is the preferred class to use for sending emails from Scrapy, as it
|
||||
uses `Twisted non-blocking IO`_, like the rest of the framework.
|
||||
uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
|
||||
rest of the framework.
|
||||
|
||||
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
|
|||
|
||||
This method is called for every item pipeline component. :meth:`process_item`
|
||||
must either: return a dict with data, return an :class:`~scrapy.item.Item`
|
||||
(or any descendant class) object, return a `Twisted Deferred`_ or raise
|
||||
(or any descendant class) object, return a
|
||||
:class:`~twisted.internet.defer.Deferred` or raise
|
||||
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
|
||||
processed by further pipeline components.
|
||||
|
||||
|
|
@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
|
|||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Item pipeline example
|
||||
=====================
|
||||
|
||||
|
|
@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
|
|||
Take screenshot of item
|
||||
-----------------------
|
||||
|
||||
This example demonstrates how to return Deferred_ from :meth:`process_item` method.
|
||||
This example demonstrates how to return a
|
||||
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
|
||||
It uses Splash_ to render screenshot of item url. Pipeline
|
||||
makes request to locally running instance of Splash_. After request is downloaded
|
||||
and Deferred callback fires, it saves item to a file and adds filename to an item.
|
||||
|
|
@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
return item
|
||||
|
||||
.. _Splash: https://splash.readthedocs.io/en/stable/
|
||||
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
Duplicates filter
|
||||
-----------------
|
||||
|
|
|
|||
|
|
@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable.
|
|||
containing the collected values (for that field). The result of the output
|
||||
processors is the value that will be finally assigned to the item.
|
||||
|
||||
If you want to use a plain function as a processor, make sure it receives
|
||||
``self`` as the first argument::
|
||||
|
||||
def lowercase_processor(self, values):
|
||||
for v in values:
|
||||
yield v.lower()
|
||||
|
||||
class MyItemLoader(ItemLoader):
|
||||
name_in = lowercase_processor
|
||||
|
||||
This is because whenever a function is assigned as a class variable, it becomes
|
||||
a method and would be passed the instance as the the first argument when being
|
||||
called. See `this answer on stackoverflow`_ for more details.
|
||||
|
||||
The other thing you need to keep in mind is that the values returned by input
|
||||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
|
@ -163,7 +149,7 @@ processors to populate the fields.
|
|||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
|
@ -491,6 +477,8 @@ ItemLoader objects
|
|||
.. attribute:: item
|
||||
|
||||
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
|
|
|
|||
|
|
@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
* ``success`` is a boolean which is ``True`` if the image was downloaded
|
||||
successfully or ``False`` if it failed for some reason
|
||||
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if success
|
||||
is ``True``) or a `Twisted Failure`_ if there was a problem.
|
||||
* ``file_info_or_error`` is a dict containing the following keys (if
|
||||
success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
|
||||
there was a problem.
|
||||
|
||||
* ``url`` - the url where the file was downloaded from. This is the url of
|
||||
the request returned from the :meth:`~get_media_requests`
|
||||
|
|
@ -577,5 +578,4 @@ above::
|
|||
item['image_paths'] = image_paths
|
||||
return item
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running.
|
|||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run() # the script will block here until the crawling is finished
|
||||
|
||||
.. seealso:: `Twisted Reactor Overview`_.
|
||||
.. seealso:: :doc:`twisted:core/howto/reactor-basics`
|
||||
|
||||
.. _run-multiple-spiders:
|
||||
|
||||
|
|
@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ Request objects
|
|||
|
||||
:param errback: a function that will be called if any exception was
|
||||
raised while processing the request. This includes pages that failed
|
||||
with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
|
||||
as first parameter.
|
||||
with 404 HTTP errors and such. It receives a
|
||||
:exc:`~twisted.python.failure.Failure` as first parameter.
|
||||
For more information,
|
||||
see :ref:`topics-request-response-ref-errbacks` below.
|
||||
:type errback: callable
|
||||
|
|
@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing
|
|||
The errback of a request is a function that will be called when an exception
|
||||
is raise while processing it.
|
||||
|
||||
It receives a `Twisted Failure`_ instance as first parameter and can be
|
||||
used to track connection establishment timeouts, DNS errors etc.
|
||||
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
|
||||
be used to track connection establishment timeouts, DNS errors etc.
|
||||
|
||||
Here's an example spider logging all errors and catching some specific
|
||||
errors if needed::
|
||||
|
|
@ -816,5 +816,4 @@ XmlResponse objects
|
|||
adds encoding auto-discovering support by looking into the XML declaration
|
||||
line. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
|
|||
Deferred signal handlers
|
||||
========================
|
||||
|
||||
Some signals support returning `Twisted deferreds`_ from their handlers, see
|
||||
the :ref:`topics-signals-ref` below to know which ones.
|
||||
Some signals support returning :class:`~twisted.internet.defer.Deferred`
|
||||
objects from their handlers, see the :ref:`topics-signals-ref` below to know
|
||||
which ones.
|
||||
|
||||
.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
.. _topics-signals-ref:
|
||||
|
||||
|
|
@ -155,8 +155,8 @@ item_error
|
|||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
spider_closed
|
||||
-------------
|
||||
|
|
@ -236,8 +236,8 @@ spider_error
|
|||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -333,5 +333,3 @@ response_downloaded
|
|||
|
||||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
|
|
|
|||
187
pytest.ini
187
pytest.ini
|
|
@ -8,8 +8,6 @@ addopts =
|
|||
--ignore=docs/_ext
|
||||
--ignore=docs/conf.py
|
||||
--ignore=docs/news.rst
|
||||
--ignore=docs/topics/commands.rst
|
||||
--ignore=docs/topics/debug.rst
|
||||
--ignore=docs/topics/developer-tools.rst
|
||||
--ignore=docs/topics/dynamic-content.rst
|
||||
--ignore=docs/topics/items.rst
|
||||
|
|
@ -22,24 +20,27 @@ addopts =
|
|||
--ignore=docs/utils
|
||||
twisted = 1
|
||||
flake8-ignore =
|
||||
# Files that are only meant to provide top-level imports are expected not
|
||||
# to use any of their imports:
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
# Issues pending a review:
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py F401 E501
|
||||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
|
||||
scrapy/commands/fetch.py E401 E501 E128 E502 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/list.py E302
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E302 E128
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
scrapy/commands/view.py F401 E302
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
|
|
@ -48,19 +49,18 @@ flake8-ignore =
|
|||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E126 E226
|
||||
scrapy/core/downloader/__init__.py F401 E501
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
|
||||
scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
|
|
@ -68,43 +68,38 @@ flake8-ignore =
|
|||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py F401 E501
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/corestats.py E302 E501
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303 F401
|
||||
scrapy/extensions/httpcache.py E128 E501 E303
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E302 E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/html.py E302
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
scrapy/http/response/xml.py E302
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/common.py E302
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E302
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403 F401
|
||||
scrapy/selector/unified.py F401 E501 E111
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
|
|
@ -112,160 +107,144 @@ flake8-ignore =
|
|||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py F401 E501 E402
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/boto.py F401
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E302 E261 F401 E306 E305
|
||||
scrapy/utils/curl.py F401
|
||||
scrapy/utils/console.py E261 E306 E305
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501 E302
|
||||
scrapy/utils/defer.py E501 E302 E128
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/display.py E302
|
||||
scrapy/utils/engine.py F401 E261 E302
|
||||
scrapy/utils/ftp.py E302
|
||||
scrapy/utils/gz.py E305 E501 E302 W504
|
||||
scrapy/utils/http.py F403 F401 E226
|
||||
scrapy/utils/httpobj.py E302 E501
|
||||
scrapy/utils/engine.py E261
|
||||
scrapy/utils/gz.py E305 E501 W504
|
||||
scrapy/utils/http.py F403 E226
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/job.py E302
|
||||
scrapy/utils/log.py E128 W503
|
||||
scrapy/utils/markup.py F403 F401 W292
|
||||
scrapy/utils/markup.py F403 W292
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/multipart.py F403 F401 W292
|
||||
scrapy/utils/multipart.py F403 W292
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501 E302
|
||||
scrapy/utils/reactor.py E302 E226
|
||||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E226
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E302 E127 E501
|
||||
scrapy/utils/response.py E501 E302 E128
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E302 E501
|
||||
scrapy/utils/spider.py E271 E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/template.py E302
|
||||
scrapy/utils/test.py E302 E501
|
||||
scrapy/utils/url.py E501 F403 F401 E128 F405
|
||||
scrapy/utils/test.py E501
|
||||
scrapy/utils/url.py E501 F403 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E302 E501 E202
|
||||
scrapy/exceptions.py E302 E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/extension.py E302
|
||||
scrapy/interfaces.py E302 E501
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E302
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E302 E501
|
||||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py F401 E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123 F401
|
||||
tests/pipelines.py E302 F841 E226
|
||||
tests/spiders.py E302 E501 E127
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/pipelines.py F841 E226
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_parse.py F401 E302 E501 E128 E303 E226
|
||||
tests/test_command_parse.py E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py F401 E128 E501
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128 W293
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py E302 F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123
|
||||
tests/test_dependencies.py F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E305
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F401 F841
|
||||
tests/test_feedexport.py E501 F401 F841 E241
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E302 E501
|
||||
tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122 E302
|
||||
tests/test_mail.py E302 E128 E501 E305
|
||||
tests/test_middleware.py E302 E501 E128
|
||||
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501 E305
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F401 F841 E501 E303
|
||||
tests/test_pipeline_files.py E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E302 E305
|
||||
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
|
||||
tests/test_responsetypes.py E501 E305
|
||||
tests/test_robotstxt_interface.py E501 W291 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py F401 E501 E127
|
||||
tests/test_spider.py E501 F401
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
|
||||
tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E302 E701 E741
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py E501 W293 E226
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E701 E741
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_console.py E302
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
|
||||
tests/test_utils_defer.py E306 E261 E501 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E302 E501 E502 E128 W504
|
||||
tests/test_utils_httpobj.py E302
|
||||
tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
|
||||
tests/test_utils_http.py E501 E502 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_reqser.py F401 E501 E128
|
||||
tests/test_utils_request.py E302 E501 E128 E305
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128 E305
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E302 E731 E226
|
||||
tests/test_utils_sitemap.py E302 E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E302 E305
|
||||
tests/test_utils_signal.py E741 F841 E731 E226
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/mocks/dummydbm.py E302
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_cmdline/extensions.py E302
|
||||
tests/test_settings/__init__.py F401 E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501 E302
|
||||
tests/test_spiderloader/test_spiders/spider0.py E302
|
||||
tests/test_spiderloader/test_spiders/spider1.py E302
|
||||
tests/test_spiderloader/test_spiders/spider2.py E302
|
||||
tests/test_spiderloader/test_spiders/spider3.py E302
|
||||
tests/test_spiderloader/test_spiders/nested/spider4.py E302
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
|||
del warnings
|
||||
|
||||
# Apply monkey patches to fix issues in external libraries
|
||||
from . import _monkeypatches
|
||||
from scrapy import _monkeypatches
|
||||
del _monkeypatches
|
||||
|
||||
from twisted import version as _txv
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from __future__ import print_function
|
||||
import time
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from unittest import TextTestRunner, TextTestResult as _TextTestResult
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import print_function
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import json
|
|||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from scrapy.commands import fetch, ScrapyCommand
|
||||
from scrapy.commands import fetch
|
||||
from scrapy.utils.response import open_in_browser
|
||||
|
||||
|
||||
class Command(fetch.Command):
|
||||
|
||||
def short_desc(self):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from scrapy.item import BaseItem
|
|||
from scrapy.http import Request
|
||||
from scrapy.exceptions import ContractFail
|
||||
|
||||
from . import Contract
|
||||
from scrapy.contracts import Contract
|
||||
|
||||
|
||||
# contracts
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from __future__ import absolute_import
|
||||
import random
|
||||
import warnings
|
||||
from time import time
|
||||
from datetime import datetime
|
||||
from collections import deque
|
||||
|
|
@ -12,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.resolver import dnscache
|
||||
from scrapy import signals
|
||||
from .middleware import DownloaderMiddlewareManager
|
||||
from .handlers import DownloadHandlers
|
||||
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
||||
|
||||
class Slot(object):
|
||||
|
|
|
|||
|
|
@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
|
||||
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
|
||||
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
The default is to use a
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
|
||||
have special requirements you can leave this as-is.
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
:meth:`creatorForNetloc` is the same as
|
||||
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
|
||||
factory allows setting the TLS/SSL method to use.
|
||||
|
||||
The default OpenSSL method is ``TLS_METHOD`` (also called
|
||||
``SSLv23_METHOD``) which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
|
|||
respcls = responsetypes.from_mimetype(uri.media_type)
|
||||
|
||||
resp_kwargs = {}
|
||||
if (issubclass(respcls, TextResponse) and
|
||||
uri.media_type.split('/')[0] == 'text'):
|
||||
if (issubclass(respcls, TextResponse)
|
||||
and uri.media_type.split('/')[0] == 'text'):
|
||||
charset = uri.media_type_parameters.get('charset')
|
||||
resp_kwargs['encoding'] = charset
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from __future__ import absolute_import
|
||||
from .http10 import HTTP10DownloadHandler
|
||||
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import (
|
||||
HTTP11DownloadHandler as HTTPDownloadHandler,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.boto import is_botocore
|
||||
from .http import HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
|
||||
|
||||
def _get_boto_connection():
|
||||
|
|
@ -21,7 +21,7 @@ def _get_boto_connection():
|
|||
return http_request.headers
|
||||
|
||||
try:
|
||||
import boto.auth
|
||||
import boto.auth # noqa: F401
|
||||
except ImportError:
|
||||
_S3Connection = _v19_S3Connection
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import pprint
|
||||
import six
|
||||
import signal
|
||||
import logging
|
||||
|
|
@ -45,7 +46,8 @@ class Crawler(object):
|
|||
logging.root.addHandler(handler)
|
||||
|
||||
d = dict(overridden_settings(self.settings))
|
||||
logger.info("Overridden settings: %(settings)r", {'settings': d})
|
||||
logger.info("Overridden settings:\n%(settings)s",
|
||||
{'settings': pprint.pformat(d)})
|
||||
|
||||
if get_scrapy_root_handler() is not None:
|
||||
# scrapy root handler already installed: update it with new settings
|
||||
|
|
@ -110,7 +112,7 @@ class Crawler(object):
|
|||
class CrawlerRunner(object):
|
||||
"""
|
||||
This is a convenient helper class that keeps track of, manages and runs
|
||||
crawlers inside an already setup Twisted `reactor`_.
|
||||
crawlers inside an already setup :mod:`~twisted.internet.reactor`.
|
||||
|
||||
The CrawlerRunner object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -233,12 +235,13 @@ class CrawlerProcess(CrawlerRunner):
|
|||
A class to run multiple scrapy crawlers in a process simultaneously.
|
||||
|
||||
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
|
||||
for starting a Twisted `reactor`_ and handling shutdown signals, like the
|
||||
keyboard interrupt command Ctrl-C. It also configures top-level logging.
|
||||
for starting a :mod:`~twisted.internet.reactor` and handling shutdown
|
||||
signals, like the keyboard interrupt command Ctrl-C. It also configures
|
||||
top-level logging.
|
||||
|
||||
This utility should be a better fit than
|
||||
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
|
||||
Twisted `reactor`_ within your application.
|
||||
:mod:`~twisted.internet.reactor` within your application.
|
||||
|
||||
The CrawlerProcess object must be instantiated with a
|
||||
:class:`~scrapy.settings.Settings` object.
|
||||
|
|
@ -273,9 +276,9 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def start(self, stop_after_crawl=True):
|
||||
"""
|
||||
This method starts a Twisted `reactor`_, adjusts its pool size to
|
||||
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
|
||||
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
|
||||
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
|
||||
based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
|
||||
|
||||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
|
|
|||
|
|
@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object):
|
|||
|
||||
# XXX: move it to w3lib?
|
||||
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
|
||||
|
||||
|
||||
def _has_ajaxcrawlable_meta(text):
|
||||
"""
|
||||
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import os
|
||||
import six
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
import six
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
from email.utils import formatdate
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import base64
|
||||
from urllib.request import _parse_proxy
|
||||
|
||||
from six.moves.urllib.parse import unquote, urlunparse
|
||||
from six.moves.urllib.request import getproxies, proxy_bypass
|
||||
from urllib.request import _parse_proxy
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
|
||||
|
||||
class BaseDupeFilter(object):
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ new exceptions here without documenting them there.
|
|||
|
||||
# Internal
|
||||
|
||||
|
||||
class NotConfigured(Exception):
|
||||
"""Indicates a missing configuration situation"""
|
||||
pass
|
||||
|
||||
|
||||
class _InvalidOutput(TypeError):
|
||||
"""
|
||||
Indicates an invalid value has been returned by a middleware's processing method.
|
||||
|
|
@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):
|
|||
"""
|
||||
pass
|
||||
|
||||
|
||||
# HTTP and crawling
|
||||
|
||||
|
||||
class IgnoreRequest(Exception):
|
||||
"""Indicates a decision was made not to process a request"""
|
||||
|
||||
|
||||
class DontCloseSpider(Exception):
|
||||
"""Request the spider not to be closed yet"""
|
||||
pass
|
||||
|
||||
|
||||
class CloseSpider(Exception):
|
||||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
||||
|
|
@ -34,30 +40,37 @@ class CloseSpider(Exception):
|
|||
super(CloseSpider, self).__init__()
|
||||
self.reason = reason
|
||||
|
||||
|
||||
# Items
|
||||
|
||||
|
||||
class DropItem(Exception):
|
||||
"""Drop item from the item pipeline"""
|
||||
pass
|
||||
|
||||
|
||||
class NotSupported(Exception):
|
||||
"""Indicates a feature or method is not supported"""
|
||||
pass
|
||||
|
||||
|
||||
# Commands
|
||||
|
||||
|
||||
class UsageError(Exception):
|
||||
"""To indicate a command-line usage error"""
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
|
||||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
"""Warning category for deprecated features, since the default
|
||||
DeprecationWarning is silenced on Python 2.7+
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class ContractFail(AssertionError):
|
||||
"""Error raised in case of a failing contract"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
|
|||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
|
||||
|
||||
class ExtensionManager(MiddlewareManager):
|
||||
|
||||
component_name = 'extension'
|
||||
|
|
|
|||
|
|
@ -6,18 +6,16 @@ import os
|
|||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from time import time
|
||||
from warnings import warn
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from scrapy import signals
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.job import job_dir
|
||||
|
||||
|
||||
class SpiderState(object):
|
||||
"""Store and load spider state during a scraping job"""
|
||||
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class HtmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
||||
|
||||
class XmlResponse(TextResponse):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from zope.interface import Interface
|
||||
|
||||
|
||||
class ISpiderLoader(Interface):
|
||||
|
||||
def from_settings(settings):
|
||||
|
|
|
|||
|
|
@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
|
|||
|
||||
|
||||
# Top-level imports
|
||||
from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from six.moves.urllib.parse import urljoin
|
|||
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
|
||||
|
||||
from scrapy.link import Link
|
||||
from .sgml import SgmlLinkExtractor
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor
|
||||
|
||||
linkre = re.compile(
|
||||
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Item Loader
|
|||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
import six
|
||||
from contextlib import suppress
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
|
|
@ -15,6 +14,17 @@ from scrapy.utils.misc import arg_to_iter, extract_regex
|
|||
from scrapy.utils.python import flatten
|
||||
|
||||
|
||||
def unbound_method(method):
|
||||
"""
|
||||
Allow to use single-argument functions as input or output processors
|
||||
(no need to define an unused first 'self' argument)
|
||||
"""
|
||||
with suppress(AttributeError):
|
||||
if '.' not in method.__qualname__:
|
||||
return method.__func__
|
||||
return method
|
||||
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
default_item_class = Item
|
||||
|
|
@ -72,7 +82,7 @@ class ItemLoader(object):
|
|||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in six.iteritems(value):
|
||||
for k, v in value.items():
|
||||
self._add_value(k, v)
|
||||
else:
|
||||
self._add_value(field_name, value)
|
||||
|
|
@ -82,7 +92,7 @@ class ItemLoader(object):
|
|||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in six.iteritems(value):
|
||||
for k, v in value.items():
|
||||
self._replace_value(k, v)
|
||||
else:
|
||||
self._replace_value(field_name, value)
|
||||
|
|
@ -142,14 +152,14 @@ class ItemLoader(object):
|
|||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return proc
|
||||
return unbound_method(proc)
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return proc
|
||||
return unbound_method(proc)
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
proc = self.get_input_processor(field_name)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from functools import partial
|
||||
from scrapy.utils.python import get_func_args
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst
|
|||
from scrapy.middleware import MiddlewareManager
|
||||
from scrapy.utils.conf import build_component_list
|
||||
|
||||
|
||||
class ItemPipelineManager(MiddlewareManager):
|
||||
|
||||
component_name = 'item pipeline'
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache
|
|||
|
||||
dnscache = LocalCache(10000)
|
||||
|
||||
|
||||
class CachingThreadedResolver(ThreadedResolver):
|
||||
def __init__(self, reactor, cache_size, timeout):
|
||||
super(CachingThreadedResolver, self).__init__(reactor)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ from six import with_metaclass
|
|||
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
|
||||
try:
|
||||
if to_native_str_type:
|
||||
|
|
@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
|
|||
robotstxt_body = ''
|
||||
return robotstxt_body
|
||||
|
||||
|
||||
class RobotParser(with_metaclass(ABCMeta)):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""
|
||||
Selectors
|
||||
"""
|
||||
from scrapy.selector.unified import *
|
||||
from scrapy.selector.unified import * # noqa: F401
|
||||
|
|
|
|||
|
|
@ -2,12 +2,10 @@
|
|||
XPath selectors based on lxml
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from parsel import Selector as _ParselSelector
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.http import HtmlResponse, XmlResponse
|
||||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
||||
__all__ = ['Selector', 'SelectorList']
|
||||
|
|
|
|||
|
|
@ -46,16 +46,14 @@ class SignalManager(object):
|
|||
|
||||
def send_catch_log_deferred(self, signal, **kwargs):
|
||||
"""
|
||||
Like :meth:`send_catch_log` but supports returning `deferreds`_ from
|
||||
signal handlers.
|
||||
Like :meth:`send_catch_log` but supports returning
|
||||
:class:`~twisted.internet.defer.Deferred` objects from signal handlers.
|
||||
|
||||
Returns a Deferred that gets fired once all signal handlers
|
||||
deferreds were fired. Send a signal, catch exceptions and log them.
|
||||
|
||||
The keyword arguments are passed to the signal handlers (connected
|
||||
through the :meth:`connect` method).
|
||||
|
||||
.. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
"""
|
||||
kwargs.setdefault('sender', self.sender)
|
||||
return _signal.send_catch_log_deferred(signal, **kwargs)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from scrapy import signals
|
|||
from scrapy.http import Request
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.deprecate import method_is_overridden
|
||||
|
||||
|
||||
|
|
@ -58,6 +57,11 @@ class Spider(object_ref):
|
|||
|
||||
def start_requests(self):
|
||||
cls = self.__class__
|
||||
if not self.start_urls and hasattr(self, 'start_url'):
|
||||
raise AttributeError(
|
||||
"Crawling could not start: 'start_urls' not found "
|
||||
"or empty (but found 'start_url' attribute instead, "
|
||||
"did you miss an 's'?)")
|
||||
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
|
||||
warnings.warn(
|
||||
"Spider.make_requests_from_url method is deprecated; it "
|
||||
|
|
@ -100,6 +104,6 @@ class Spider(object_ref):
|
|||
|
||||
|
||||
# Top-level imports
|
||||
from scrapy.spiders.crawl import CrawlSpider, Rule
|
||||
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
|
||||
from scrapy.spiders.sitemap import SitemapSpider
|
||||
from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401
|
||||
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401
|
||||
from scrapy.spiders.sitemap import SitemapSpider # noqa: F401
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from scrapy.exceptions import NotConfigured
|
|||
|
||||
def is_botocore():
|
||||
try:
|
||||
import botocore
|
||||
import botocore # noqa: F401
|
||||
return True
|
||||
except ImportError:
|
||||
raise NotConfigured('missing botocore library')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from functools import wraps
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def _embed_ipython_shell(namespace={}, banner=''):
|
||||
"""Start an IPython Shell"""
|
||||
try:
|
||||
|
|
@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
|
|||
shell()
|
||||
return wrapper
|
||||
|
||||
|
||||
def _embed_bpython_shell(namespace={}, banner=''):
|
||||
"""Start a bpython shell"""
|
||||
import bpython
|
||||
|
|
@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
|
|||
bpython.embed(locals_=namespace, banner=banner)
|
||||
return wrapper
|
||||
|
||||
|
||||
def _embed_ptpython_shell(namespace={}, banner=''):
|
||||
"""Start a ptpython shell"""
|
||||
import ptpython.repl
|
||||
|
|
@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
|
|||
ptpython.repl.embed(locals=namespace)
|
||||
return wrapper
|
||||
|
||||
|
||||
def _embed_standard_shell(namespace={}, banner=''):
|
||||
"""Start a standard python shell"""
|
||||
import code
|
||||
|
|
@ -48,13 +52,14 @@ def _embed_standard_shell(namespace={}, banner=''):
|
|||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
import rlcompleter
|
||||
import rlcompleter # noqa: F401
|
||||
readline.parse_and_bind("tab:complete")
|
||||
@wraps(_embed_standard_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
code.interact(banner=banner, local=namespace)
|
||||
return wrapper
|
||||
|
||||
|
||||
DEFAULT_PYTHON_SHELLS = OrderedDict([
|
||||
('ptpython', _embed_ptpython_shell),
|
||||
('ipython', _embed_ipython_shell),
|
||||
|
|
@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([
|
|||
('python', _embed_standard_shell),
|
||||
])
|
||||
|
||||
|
||||
def get_shell_embed_func(shells=None, known_shells=None):
|
||||
"""Return the first acceptable shell-embed function
|
||||
from a given list of shell names.
|
||||
|
|
@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None):
|
|||
except ImportError:
|
||||
continue
|
||||
|
||||
|
||||
def start_python_console(namespace=None, banner='', shells=None):
|
||||
"""Start Python console bound to the given namespace.
|
||||
Readline support and tab completion will be used on Unix, if available.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from shlex import split
|
|||
|
||||
from six.moves.http_cookies import SimpleCookie
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from six import string_types, iteritems
|
||||
from six import iteritems
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ def defers(func):
|
|||
return defer.maybeDeferred(func, *a, **kw)
|
||||
return wrapped
|
||||
|
||||
|
||||
def inthread(func):
|
||||
"""Decorator to call a function in a thread and return a deferred with the
|
||||
result
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from twisted.python import failure
|
|||
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
|
||||
|
||||
def defer_fail(_failure):
|
||||
"""Same as twisted.internet.defer.fail but delay calling errback until
|
||||
next reactor loop
|
||||
|
|
@ -18,6 +19,7 @@ def defer_fail(_failure):
|
|||
reactor.callLater(0.1, d.errback, _failure)
|
||||
return d
|
||||
|
||||
|
||||
def defer_succeed(result):
|
||||
"""Same as twisted.internet.defer.succeed but delay calling callback until
|
||||
next reactor loop
|
||||
|
|
@ -29,6 +31,7 @@ def defer_succeed(result):
|
|||
reactor.callLater(0.1, d.callback, result)
|
||||
return d
|
||||
|
||||
|
||||
def defer_result(result):
|
||||
if isinstance(result, defer.Deferred):
|
||||
return result
|
||||
|
|
@ -37,6 +40,7 @@ def defer_result(result):
|
|||
else:
|
||||
return defer_succeed(result)
|
||||
|
||||
|
||||
def mustbe_deferred(f, *args, **kw):
|
||||
"""Same as twisted.internet.defer.maybeDeferred, but delay calling
|
||||
callback/errback to next reactor loop
|
||||
|
|
@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw):
|
|||
else:
|
||||
return defer_result(result)
|
||||
|
||||
|
||||
def parallel(iterable, count, callable, *args, **named):
|
||||
"""Execute a callable over the objects in the given iterable, in parallel,
|
||||
using no more than ``count`` concurrent calls.
|
||||
|
|
@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named):
|
|||
work = (callable(elem, *args, **named) for elem in iterable)
|
||||
return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
|
||||
|
||||
|
||||
def process_chain(callbacks, input, *a, **kw):
|
||||
"""Return a Deferred built by chaining the given callbacks"""
|
||||
d = defer.Deferred()
|
||||
|
|
@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw):
|
|||
d.callback(input)
|
||||
return d
|
||||
|
||||
|
||||
def process_chain_both(callbacks, errbacks, input, *a, **kw):
|
||||
"""Return a Deferred built by chaining the given callbacks and errbacks"""
|
||||
d = defer.Deferred()
|
||||
|
|
@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
|
|||
d.callback(input)
|
||||
return d
|
||||
|
||||
|
||||
def process_parallel(callbacks, input, *a, **kw):
|
||||
"""Return a Deferred with the output of all successful calls to the given
|
||||
callbacks
|
||||
|
|
@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw):
|
|||
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
|
||||
return d
|
||||
|
||||
|
||||
def iter_errback(iterable, errback, *a, **kw):
|
||||
"""Wraps an iterable calling an errback if an error is caught while
|
||||
iterating it.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from __future__ import print_function
|
|||
import sys
|
||||
from pprint import pformat as pformat_
|
||||
|
||||
|
||||
def _colorize(text, colorize=True):
|
||||
if not colorize or not sys.stdout.isatty():
|
||||
return text
|
||||
|
|
@ -17,8 +18,10 @@ def _colorize(text, colorize=True):
|
|||
except ImportError:
|
||||
return text
|
||||
|
||||
|
||||
def pformat(obj, *args, **kwargs):
|
||||
return _colorize(pformat_(obj), kwargs.pop('colorize', True))
|
||||
|
||||
|
||||
def pprint(obj, *args, **kwargs):
|
||||
print(pformat(obj, *args, **kwargs))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
"""Some debugging functions for working with the Scrapy engine"""
|
||||
|
||||
from __future__ import print_function
|
||||
from time import time # used in global tests code
|
||||
# used in global tests code
|
||||
from time import time # noqa: F401
|
||||
|
||||
|
||||
def get_engine_status(engine):
|
||||
"""Return a report of the current engine status"""
|
||||
|
|
@ -32,6 +33,7 @@ def get_engine_status(engine):
|
|||
|
||||
return checks
|
||||
|
||||
|
||||
def format_engine_status(engine=None):
|
||||
checks = get_engine_status(engine)
|
||||
s = "Execution engine status\n\n"
|
||||
|
|
@ -41,5 +43,6 @@ def format_engine_status(engine=None):
|
|||
|
||||
return s
|
||||
|
||||
|
||||
def print_engine_status(engine):
|
||||
print(format_engine_status(engine))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from ftplib import error_perm
|
||||
from posixpath import dirname
|
||||
|
||||
|
||||
def ftp_makedirs_cwd(ftp, path, first_call=True):
|
||||
"""Set the current directory of the FTP connection given in the ``ftp``
|
||||
argument (as a ftplib.FTP object), creating all parent directories if they
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ def gunzip(data):
|
|||
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
|
||||
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
|
||||
|
||||
|
||||
@deprecated
|
||||
def is_gzipped(response):
|
||||
"""Return True if the response is gzipped, or False otherwise"""
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import warnings
|
|||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from w3lib.http import *
|
||||
from w3lib.http import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.http` is deprecated, "
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@ import weakref
|
|||
|
||||
from six.moves.urllib.parse import urlparse
|
||||
|
||||
|
||||
_urlparse_cache = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def urlparse_cached(request_or_response):
|
||||
"""Return urlparse.urlparse caching the result, where the argument can be a
|
||||
Request or Response object
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
|
||||
|
||||
def job_dir(settings):
|
||||
path = settings['JOBDIR']
|
||||
if path and not os.path.exists(path):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this module
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from w3lib.html import *
|
||||
from w3lib.html import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.markup` is deprecated. "
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this module
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from w3lib.form import *
|
||||
from w3lib.form import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ def memoizemethod_noargs(method):
|
|||
_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
|
||||
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
|
||||
|
||||
|
||||
@deprecated("scrapy.utils.python.binary_is_text")
|
||||
def isbinarytext(text):
|
||||
""" This function is deprecated.
|
||||
|
|
@ -382,9 +383,11 @@ class MutableChain(object):
|
|||
self.data = chain(self.data, *iterables)
|
||||
|
||||
def __iter__(self):
|
||||
return self.data.__iter__()
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return next(self.data)
|
||||
|
||||
next = __next__
|
||||
@deprecated("scrapy.utils.python.MutableChain.__next__")
|
||||
def next(self):
|
||||
return self.__next__()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from twisted.internet import reactor, error
|
||||
|
||||
|
||||
def listen_tcp(portrange, host, factory):
|
||||
"""Like reactor.listenTCP but tries different ports in a range."""
|
||||
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
|
||||
|
||||
_fingerprint_cache = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
||||
"""
|
||||
Return the request fingerprint.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ from w3lib import html
|
|||
|
||||
|
||||
_baseurl_cache = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def get_base_url(response):
|
||||
"""Return the base url of the given response, joined with the response url"""
|
||||
if response not in _baseurl_cache:
|
||||
|
|
@ -23,6 +25,8 @@ def get_base_url(response):
|
|||
|
||||
|
||||
_metaref_cache = weakref.WeakKeyDictionary()
|
||||
|
||||
|
||||
def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
|
||||
"""Parse the http-equiv refrsh parameter from the given response"""
|
||||
if response not in _metaref_cache:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ def iter_spider_classes(module):
|
|||
getattr(obj, 'name', None):
|
||||
yield obj
|
||||
|
||||
|
||||
def spidercls_for_request(spider_loader, request, default_spidercls=None,
|
||||
log_none=False, log_multiple=False):
|
||||
"""Return a spider class that handles the given Request.
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs):
|
|||
|
||||
|
||||
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
|
||||
|
||||
|
||||
def string_camelcase(string):
|
||||
""" Convert a word to its CamelCase version and remove invalid chars
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ def skip_if_no_boto():
|
|||
except NotConfigured as e:
|
||||
raise SkipTest(e)
|
||||
|
||||
|
||||
def get_s3_content_and_delete(bucket, path, with_key=False):
|
||||
""" Get content from s3 key, and delete key afterwards.
|
||||
"""
|
||||
|
|
@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
|
|||
bucket.delete_key(path)
|
||||
return (content, key) if with_key else content
|
||||
|
||||
|
||||
def get_gcs_content_and_delete(bucket, path):
|
||||
from google.cloud import storage
|
||||
client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
|
||||
|
|
@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path):
|
|||
bucket.delete_blob(path)
|
||||
return content, acl, blob
|
||||
|
||||
|
||||
def get_crawler(spidercls=None, settings_dict=None):
|
||||
"""Return an unconfigured Crawler object. If settings_dict is given, it
|
||||
will be used to populate the crawler settings with a project level
|
||||
|
|
@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None):
|
|||
runner = CrawlerRunner(settings_dict)
|
||||
return runner.create_crawler(spidercls or Spider)
|
||||
|
||||
|
||||
def get_pythonpath():
|
||||
"""Return a PYTHONPATH suitable to use in processes so that they find this
|
||||
installation of Scrapy"""
|
||||
scrapy_path = import_module('scrapy').__path__[0]
|
||||
return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
|
||||
|
||||
|
||||
def get_testenv():
|
||||
"""Return a OS environment dict suitable to fork processes that need to import
|
||||
this installation of Scrapy, instead of a system installed one.
|
||||
|
|
@ -86,6 +91,7 @@ def get_testenv():
|
|||
env['PYTHONPATH'] = get_pythonpath()
|
||||
return env
|
||||
|
||||
|
||||
def assert_samelines(testcase, text1, text2, msg=None):
|
||||
"""Asserts text1 and text2 have the same lines, ignoring differences in
|
||||
line endings between platforms
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse
|
|||
# scrapy.utils.url was moved to w3lib.url and import * ensures this
|
||||
# move doesn't break old code
|
||||
from w3lib.url import *
|
||||
from w3lib.url import _safe_chars, _unquotepath
|
||||
from w3lib.url import _safe_chars, _unquotepath # noqa: F401
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,5 +27,5 @@ def scrapy_components_versions():
|
|||
("Python", sys.version.replace("\n", "- ")),
|
||||
("pyOpenSSL", get_openssl_version()),
|
||||
("cryptography", cryptography.__version__),
|
||||
("Platform", platform.platform()),
|
||||
("Platform", platform.platform()),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
tests/test_linkextractors_deprecated.py
|
||||
tests/test_proxy_connect.py
|
||||
|
||||
scrapy/linkextractors/sgml.py
|
||||
scrapy/linkextractors/regex.py
|
||||
scrapy/linkextractors/htmlparser.py
|
||||
|
|
@ -13,6 +13,7 @@ error = KeyError
|
|||
|
||||
_DATABASES = collections.defaultdict(DummyDB)
|
||||
|
||||
|
||||
def open(file, flag='r', mode=0o666):
|
||||
"""Open or create a dummy database compatible.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
from __future__ import print_function
|
||||
import sys, time, random, os, json
|
||||
from six.moves.urllib.parse import urlencode
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
from OpenSSL import SSL
|
||||
from six.moves.urllib.parse import urlencode
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
from twisted.web.resource import Resource
|
||||
from twisted.web.static import File
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Some pipelines used for testing
|
||||
"""
|
||||
|
||||
|
||||
class ZeroDivisionErrorPipeline(object):
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
# Tests requirements
|
||||
jmespath
|
||||
mitmproxy; python_version >= '3.6'
|
||||
mitmproxy==3.0.4; python_version < '3.6'
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-twisted
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ class MockServerSpider(Spider):
|
|||
super(MockServerSpider, self).__init__(*args, **kwargs)
|
||||
self.mockserver = mockserver
|
||||
|
||||
|
||||
class MetaSpider(MockServerSpider):
|
||||
|
||||
name = 'meta'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""A test extension used to check the settings loading order"""
|
||||
|
||||
|
||||
class TestExtension(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ def _textmode(bstr):
|
|||
and reading from it in text mode"""
|
||||
return to_unicode(bstr).replace(os.linesep, '\n')
|
||||
|
||||
|
||||
class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
|
||||
command = 'parse'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from importlib import import_module
|
||||
from twisted.trial import unittest
|
||||
|
||||
|
||||
class ScrapyUtilsTest(unittest.TestCase):
|
||||
def test_required_openssl_version(self):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import six
|
||||
import shutil
|
||||
import tempfile
|
||||
from unittest import mock
|
||||
|
|
@ -543,7 +542,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
|
|||
|
||||
def setUp(self):
|
||||
try:
|
||||
from service_identity.exceptions import CertificateError
|
||||
from service_identity.exceptions import CertificateError # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("cryptography lib is too old")
|
||||
self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
|
||||
|
|
@ -630,18 +629,16 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
# download_maxsize < 100, hence the CancelledError
|
||||
self.assertIsInstance(failure.value, defer.CancelledError)
|
||||
|
||||
if six.PY2:
|
||||
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
|
||||
request = request.replace(url=self.mockserver.url('/xpayload'))
|
||||
yield crawler.crawl(seed=request)
|
||||
# download_maxsize = 50 is enough for the gzipped response
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
self.assertTrue(failure is None)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertTrue(reason, 'finished')
|
||||
else:
|
||||
# See issue https://twistedmatrix.com/trac/ticket/8175
|
||||
raise unittest.SkipTest("xpayload only enabled for PY2")
|
||||
# See issue https://twistedmatrix.com/trac/ticket/8175
|
||||
raise unittest.SkipTest("xpayload fails on PY3")
|
||||
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
|
||||
request = request.replace(url=self.mockserver.url('/xpayload'))
|
||||
yield crawler.crawl(seed=request)
|
||||
# download_maxsize = 50 is enough for the gzipped response
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
self.assertTrue(failure is None)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertTrue(reason, 'finished')
|
||||
|
||||
|
||||
class UriResource(resource.Resource):
|
||||
|
|
@ -778,7 +775,7 @@ class S3TestCase(unittest.TestCase):
|
|||
@contextlib.contextmanager
|
||||
def _mocked_date(self, date):
|
||||
try:
|
||||
import botocore.auth
|
||||
import botocore.auth # noqa: F401
|
||||
except ImportError:
|
||||
yield
|
||||
else:
|
||||
|
|
@ -843,8 +840,10 @@ class S3TestCase(unittest.TestCase):
|
|||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
try: import botocore
|
||||
except ImportError: pass
|
||||
try:
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
raise unittest.SkipTest(
|
||||
'botocore does not support overriding date with x-amz-date')
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ from scrapy.spiders import Spider
|
|||
from scrapy.http import Request, HtmlResponse, Response
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']
|
||||
|
||||
|
||||
class AjaxCrawlMiddlewareTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
from __future__ import print_function
|
||||
import time
|
||||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
import email.utils
|
||||
from contextlib import contextmanager
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
from scrapy.http import Response, HtmlResponse, Request
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -149,6 +146,7 @@ class FilesystemStorageTest(DefaultStorageTest):
|
|||
|
||||
storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
|
||||
|
||||
class FilesystemStorageGzipTest(FilesystemStorageTest):
|
||||
|
||||
def _get_settings(self, **new_settings):
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase):
|
|||
|
||||
def test_process_response_br(self):
|
||||
try:
|
||||
import brotli
|
||||
import brotli # noqa: F401
|
||||
except ImportError:
|
||||
raise SkipTest("no brotli")
|
||||
response = self._getresponse('br')
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
import os
|
||||
import sys
|
||||
from functools import partial
|
||||
from twisted.trial.unittest import TestCase, SkipTest
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.http import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import Settings
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir
|
|||
from scrapy.utils.test import get_crawler
|
||||
from tests.spiders import SimpleSpider
|
||||
|
||||
|
||||
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocol
|
|||
from twisted.cred import credentials
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.extensions.telnet import TelnetConsole, logger
|
||||
from scrapy.extensions.telnet import TelnetConsole
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
create=True)
|
||||
def test_parse_credentials(self):
|
||||
try:
|
||||
import boto
|
||||
import boto # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("S3FeedStorage requires boto")
|
||||
aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
|
||||
|
|
@ -268,7 +268,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_store_botocore_without_acl(self):
|
||||
try:
|
||||
import botocore
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest('botocore is required')
|
||||
|
||||
|
|
@ -288,7 +288,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_store_botocore_with_acl(self):
|
||||
try:
|
||||
import botocore
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest('botocore is required')
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import copy
|
|||
|
||||
from scrapy.http import Headers
|
||||
|
||||
|
||||
class HeadersTest(unittest.TestCase):
|
||||
|
||||
def assertSortedEqual(self, first, second, msg=None):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import cgi
|
||||
import unittest
|
||||
import re
|
||||
import json
|
||||
|
|
@ -8,7 +6,7 @@ from urllib.parse import unquote_to_bytes
|
|||
import warnings
|
||||
|
||||
from six.moves import xmlrpc_client as xmlrpclib
|
||||
from six.moves.urllib.parse import urlparse, parse_qs, unquote
|
||||
from six.moves.urllib.parse import urlparse, parse_qs
|
||||
|
||||
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
|
|
|||
|
|
@ -1,233 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
from scrapy.linkextractors.regex import RegexLinkExtractor
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
|
||||
from tests import get_testdata
|
||||
|
||||
from tests.test_linkextractors import Base
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
|
||||
# XXX: should we move some of these tests to base link extractor tests?
|
||||
|
||||
def test_basic(self):
|
||||
html = """<html><head><title>Page title<title>
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
<p><a href="/about.html">About us</a></p>
|
||||
<img src="/logo.png" alt="Company logo (not a link)" />
|
||||
<p><a href="../othercat.html">Other category</a></p>
|
||||
<p><a href="/">>></a></p>
|
||||
<p><a href="/" /></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
|
||||
Link(url='http://example.org/about.html', text='About us'),
|
||||
Link(url='http://example.org/othercat.html', text='Other category'),
|
||||
Link(url='http://example.org/', text='>>'),
|
||||
Link(url='http://example.org/', text='')])
|
||||
|
||||
def test_base_url(self):
|
||||
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
# base url is an absolute path and relative to host
|
||||
html = """<html><head><title>Page title<title><base href="/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://example.org/item/12.html', text='Item 12')])
|
||||
|
||||
# base url has no scheme
|
||||
html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
|
||||
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
|
||||
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
|
||||
|
||||
def test_link_text_wrong_encoding(self):
|
||||
html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
|
||||
response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
|
||||
])
|
||||
|
||||
def test_extraction_encoding(self):
|
||||
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
|
||||
response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
|
||||
response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
|
||||
body = get_testdata('link_extractor', 'linkextractor_latin1.html')
|
||||
response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response_utf8), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_noenc), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
# document encoding does not affect URL path component, only query part
|
||||
# >>> u'sample_ñ.html'.encode('utf8')
|
||||
# b'sample_\xc3\xb1.html'
|
||||
# >>> u"sample_á.html".encode('utf8')
|
||||
# b'sample_\xc3\xa1.html'
|
||||
# >>> u"sample_ö.html".encode('utf8')
|
||||
# b'sample_\xc3\xb6.html'
|
||||
# >>> u"£32".encode('latin1')
|
||||
# b'\xa332'
|
||||
# >>> u"µ".encode('latin1')
|
||||
# b'\xb5'
|
||||
self.assertEqual(lx.extract_links(response_latin1), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
|
||||
Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
|
||||
])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
url2 = 'http://evenmorestuff.com/uglystuff/index'
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.matches(url1), True)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
|
||||
class HtmlParserLinkExtractorTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
|
||||
])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
||||
extractor_cls = SgmlLinkExtractor
|
||||
escapes_whitespace = True
|
||||
|
||||
def test_deny_extensions(self):
|
||||
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = SgmlLinkExtractor(deny_extensions="jpg")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
def test_attrs_sgml(self):
|
||||
html = """<html><area href="sample1.html"></area>
|
||||
<a ref="sample2.html">sample text 2</a></html>"""
|
||||
response = HtmlResponse("http://example.com/index.html", body=html)
|
||||
lx = SgmlLinkExtractor(attrs="href")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
])
|
||||
|
||||
def test_link_nofollow(self):
|
||||
html = """
|
||||
<a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
|
||||
<a href="about.html">About us</a>
|
||||
<a href="http://google.com/something" rel="external nofollow">Something</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/page.html", body=html)
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'Something', nofollow=True),
|
||||
])
|
||||
|
||||
|
||||
class RegexLinkExtractorTestCase(unittest.TestCase):
|
||||
# XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
|
||||
# not to depend on SgmlLinkExractor. Its speed is also much worse
|
||||
# than it should be.
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
self.response = HtmlResponse(url='http://example.com/index', body=body)
|
||||
|
||||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
<a href="http://example.org/item1.html">Item 1</a>
|
||||
<a href="http://[example.org/item2.html">Item 2</a>
|
||||
<a href="http://example.org/item3.html">Item 3</a>
|
||||
"""
|
||||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
def test_html_base_href(self):
|
||||
html = """
|
||||
<html>
|
||||
<head>
|
||||
<base href="http://b.com/">
|
||||
</head>
|
||||
<body>
|
||||
<a href="test.html"></a>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
response = HtmlResponse("http://a.com/", body=html)
|
||||
lx = RegexLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://b.com/test.html', text=u'', nofollow=False),
|
||||
])
|
||||
|
||||
@unittest.expectedFailure
|
||||
def test_extraction(self):
|
||||
# RegexLinkExtractor doesn't parse URLs with leading/trailing
|
||||
# whitespaces correctly.
|
||||
super(RegexLinkExtractorTestCase, self).test_extraction()
|
||||
|
|
@ -994,5 +994,53 @@ class SelectJmesTestCase(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
# Functions as processors
|
||||
|
||||
def function_processor_strip(iterable):
|
||||
return [x.strip() for x in iterable]
|
||||
|
||||
|
||||
def function_processor_upper(iterable):
|
||||
return [x.upper() for x in iterable]
|
||||
|
||||
|
||||
class FunctionProcessorItem(Item):
|
||||
foo = Field(
|
||||
input_processor=function_processor_strip,
|
||||
output_processor=function_processor_upper,
|
||||
)
|
||||
|
||||
|
||||
class FunctionProcessorItemLoader(ItemLoader):
|
||||
default_item_class = FunctionProcessorItem
|
||||
|
||||
|
||||
class FunctionProcessorDictLoader(ItemLoader):
|
||||
default_item_class = dict
|
||||
foo_in = function_processor_strip
|
||||
foo_out = function_processor_upper
|
||||
|
||||
|
||||
class FunctionProcessorTestCase(unittest.TestCase):
|
||||
|
||||
def test_processor_defined_in_item(self):
|
||||
lo = FunctionProcessorItemLoader()
|
||||
lo.add_value('foo', ' bar ')
|
||||
lo.add_value('foo', [' asdf ', ' qwerty '])
|
||||
self.assertEqual(
|
||||
dict(lo.load_item()),
|
||||
{'foo': ['BAR', 'ASDF', 'QWERTY']}
|
||||
)
|
||||
|
||||
def test_processor_defined_in_item_loader(self):
|
||||
lo = FunctionProcessorDictLoader()
|
||||
lo.add_value('foo', ' bar ')
|
||||
lo.add_value('foo', [' asdf ', ' qwerty '])
|
||||
self.assertEqual(
|
||||
dict(lo.load_item()),
|
||||
{'foo': ['BAR', 'ASDF', 'QWERTY']}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ class DropSomeItemsPipeline(object):
|
|||
else:
|
||||
self.drop = True
|
||||
|
||||
|
||||
class ShowOrSkipMessagesTestCase(TwistedTestCase):
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from email.charset import Charset
|
|||
|
||||
from scrapy.mail import MailSender
|
||||
|
||||
|
||||
class MailSenderTest(unittest.TestCase):
|
||||
|
||||
def test_send(self):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from scrapy.settings import Settings
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
|
||||
|
||||
class M1(object):
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -15,6 +16,7 @@ class M1(object):
|
|||
def process(self, response, request, spider):
|
||||
pass
|
||||
|
||||
|
||||
class M2(object):
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -25,6 +27,7 @@ class M2(object):
|
|||
|
||||
pass
|
||||
|
||||
|
||||
class M3(object):
|
||||
|
||||
def process(self, response, request, spider):
|
||||
|
|
@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager):
|
|||
if hasattr(mw, 'process'):
|
||||
self.methods['process'].append(mw.process)
|
||||
|
||||
|
||||
class MiddlewareManagerTest(unittest.TestCase):
|
||||
|
||||
def test_init(self):
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue