mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into response_ip_address
This commit is contained in:
Eugenio Lacuesta
commit
1f2e2a6006
|
|
@ -295,3 +295,10 @@ intersphinx_mapping = {
|
|||
# ------------------------------------
|
||||
|
||||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@ Scrapy.
|
|||
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
|
||||
|
||||
If you're new to programming and want to start with Python, the following books
|
||||
may be useful to you:
|
||||
may be useful to you:
|
||||
|
||||
* `Automate the Boring Stuff With Python`_
|
||||
|
||||
* `How To Think Like a Computer Scientist`_
|
||||
* `How To Think Like a Computer Scientist`_
|
||||
|
||||
* `Learn Python 3 The Hard Way`_
|
||||
* `Learn Python 3 The Hard Way`_
|
||||
|
||||
You can also take a look at `this list of Python resources for non-programmers`_,
|
||||
as well as the `suggested resources in the learnpython-subreddit`_.
|
||||
as well as the `suggested resources in the learnpython-subreddit`_.
|
||||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
|
|
@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
|
|||
__init__.py
|
||||
|
||||
items.py # project items definition file
|
||||
|
||||
|
||||
middlewares.py # project middlewares file
|
||||
|
||||
pipelines.py # project pipelines file
|
||||
|
|
@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data.
|
|||
|
||||
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
|
||||
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
|
||||
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
|
||||
expressions`_:
|
||||
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
|
||||
:doc:`regular expressions <library/re>`:
|
||||
|
||||
>>> response.css('title::text').re(r'Quotes.*')
|
||||
['Quotes to Scrape']
|
||||
|
|
@ -305,7 +305,6 @@ with a selector (see :ref:`topics-developer-tools`).
|
|||
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
|
||||
visually selected elements, which works in many browsers.
|
||||
|
||||
.. _regular expressions: https://docs.python.org/3/library/re.html
|
||||
.. _Selector Gadget: https://selectorgadget.com/
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,22 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.0.1:
|
||||
|
||||
Scrapy 2.0.1 (2020-03-18)
|
||||
-------------------------
|
||||
|
||||
* :meth:`Response.follow_all <scrapy.http.Response.follow_all>` now supports
|
||||
an empty URL iterable as input (:issue:`4408`, :issue:`4420`)
|
||||
|
||||
* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent
|
||||
errors about the wrong Twisted reactor being installed when setting a
|
||||
different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`,
|
||||
:issue:`4406`)
|
||||
|
||||
* Fixed tests (:issue:`4422`)
|
||||
|
||||
|
||||
.. _release-2.0.0:
|
||||
|
||||
Scrapy 2.0.0 (2020-03-03)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Sphinx>=2.1
|
||||
sphinx-hoverxref
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
Sphinx>=3.0
|
||||
sphinx-hoverxref>=0.2b1
|
||||
sphinx-notfound-page>=0.4
|
||||
sphinx_rtd_theme>=0.4
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ Detecting check runs
|
|||
====================
|
||||
|
||||
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
|
||||
set to the ``true`` string. You can use `os.environ`_ to perform any change to
|
||||
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
|
||||
your spiders or your settings when ``scrapy check`` is used::
|
||||
|
||||
import os
|
||||
|
|
@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used::
|
|||
def __init__(self):
|
||||
if os.environ.get('SCRAPY_CHECK'):
|
||||
pass # Do some scraper adjustments when a check is running
|
||||
|
||||
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ becomes::
|
|||
|
||||
Coroutines may be used to call asynchronous code. This includes other
|
||||
coroutines, functions that return Deferreds and functions that return
|
||||
`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
|
||||
many useful Python libraries providing such code::
|
||||
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
|
||||
This means you can use many useful Python libraries providing such code::
|
||||
|
||||
class MySpider(Spider):
|
||||
# ...
|
||||
|
|
@ -107,4 +107,3 @@ Common use cases for asynchronous code include:
|
|||
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
|
||||
|
||||
.. _aio-libs: https://github.com/aio-libs
|
||||
.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable
|
||||
|
|
|
|||
|
|
@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that
|
|||
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments.
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
As you can see, with a few inspections in the `Network`-tool we
|
||||
were able to easily replicate the dynamic requests of the scrolling
|
||||
functionality of the page. Crawling dynamic pages can be quite
|
||||
|
|
|
|||
|
|
@ -739,7 +739,7 @@ HttpProxyMiddleware
|
|||
This middleware sets the HTTP proxy to use for requests, by setting the
|
||||
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
|
||||
|
||||
Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
|
||||
Like the Python standard library module :mod:`urllib.request`, it obeys
|
||||
the following environment variables:
|
||||
|
||||
* ``http_proxy``
|
||||
|
|
@ -751,9 +751,6 @@ HttpProxyMiddleware
|
|||
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
|
||||
environment variables, and it will also ignore ``no_proxy`` environment variable.
|
||||
|
||||
.. _urllib: https://docs.python.org/2/library/urllib.html
|
||||
.. _urllib2: https://docs.python.org/2/library/urllib2.html
|
||||
|
||||
RedirectMiddleware
|
||||
------------------
|
||||
|
||||
|
|
@ -829,6 +826,7 @@ REDIRECT_MAX_TIMES
|
|||
Default: ``20``
|
||||
|
||||
The maximum number of redirections that will be followed for a single request.
|
||||
After this maximum, the request's response is returned as is.
|
||||
|
||||
MetaRefreshMiddleware
|
||||
---------------------
|
||||
|
|
@ -1036,8 +1034,7 @@ Scrapy uses this parser by default.
|
|||
RobotFileParser
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Based on `RobotFileParser
|
||||
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
|
||||
Based on :class:`~urllib.robotparser.RobotFileParser`:
|
||||
|
||||
* is Python's built-in robots.txt_ parser
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
|
|||
probably not your request, but the target server. The target server might be
|
||||
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
.. _topics-handling-response-formats:
|
||||
|
||||
Handling different response formats
|
||||
|
|
@ -115,7 +118,7 @@ data from it depends on the type of response:
|
|||
- If the response is HTML or XML, use :ref:`selectors
|
||||
<topics-selectors>` as usual.
|
||||
|
||||
- If the response is JSON, use `json.loads`_ to load the desired data from
|
||||
- If the response is JSON, use :func:`json.loads` to load the desired data from
|
||||
:attr:`response.text <scrapy.http.TextResponse.text>`::
|
||||
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -130,8 +133,9 @@ data from it depends on the type of response:
|
|||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||
|
||||
- If the response is CSS, use a `regular expression`_ to extract the desired
|
||||
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
|
||||
- If the response is CSS, use a :doc:`regular expression <library/re>` to
|
||||
extract the desired data from
|
||||
:attr:`response.text <scrapy.http.TextResponse.text>`.
|
||||
|
||||
.. _topics-parsing-images:
|
||||
|
||||
|
|
@ -168,8 +172,9 @@ JavaScript code:
|
|||
Once you have a string with the JavaScript code, you can extract the desired
|
||||
data from it:
|
||||
|
||||
- You might be able to use a `regular expression`_ to extract the desired
|
||||
data in JSON format, which you can then parse with `json.loads`_.
|
||||
- You might be able to use a :doc:`regular expression <library/re>` to
|
||||
extract the desired data in JSON format, which you can then parse with
|
||||
:func:`json.loads`.
|
||||
|
||||
For example, if the JavaScript code contains a separate line like
|
||||
``var data = {"field": "value"};`` you can extract that data as follows:
|
||||
|
|
@ -241,9 +246,7 @@ along with `scrapy-selenium`_ for seamless integration.
|
|||
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
|
||||
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
|
||||
.. _js2xml: https://github.com/scrapinghub/js2xml
|
||||
.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
|
||||
.. _pytesseract: https://github.com/madmaze/pytesseract
|
||||
.. _regular expression: https://docs.python.org/3/library/re.html
|
||||
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
.. _Selenium: https://www.selenium.dev/
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Sending e-mail
|
|||
.. module:: scrapy.mail
|
||||
:synopsis: Email sending facility
|
||||
|
||||
Although Python makes sending e-mails relatively easy via the `smtplib`_
|
||||
Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
|
||||
library, Scrapy provides its own facility for sending e-mails which is very
|
||||
easy to use and it's implemented using :doc:`Twisted non-blocking IO
|
||||
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
|
||||
|
|
@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
|
|||
it's very easy to configure, with a few :ref:`settings
|
||||
<topics-email-settings>`.
|
||||
|
||||
.. _smtplib: https://docs.python.org/2/library/smtplib.html
|
||||
|
||||
Quick example
|
||||
=============
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ value of one of their fields::
|
|||
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
|
||||
class PerYearXmlExportPipeline(object):
|
||||
class PerYearXmlExportPipeline:
|
||||
"""Distribute items across multiple XML files according to their 'year' field"""
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -311,7 +311,7 @@ CsvItemExporter
|
|||
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -320,8 +320,6 @@ CsvItemExporter
|
|||
Color TV,1200
|
||||
DVD player,200
|
||||
|
||||
.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
|
||||
|
||||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
|
|
@ -335,15 +333,13 @@ PickleItemExporter
|
|||
:param protocol: The pickle protocol to use.
|
||||
:type protocol: int
|
||||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
For more information, see :mod:`pickle`.
|
||||
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
|
||||
|
||||
PprintItemExporter
|
||||
------------------
|
||||
|
||||
|
|
@ -372,8 +368,8 @@ JsonItemExporter
|
|||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -393,8 +389,6 @@ JsonItemExporter
|
|||
stream-friendly format, consider using :class:`JsonLinesItemExporter`
|
||||
instead, or splitting the output in multiple chunks.
|
||||
|
||||
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
|
||||
|
||||
JsonLinesItemExporter
|
||||
---------------------
|
||||
|
||||
|
|
@ -403,8 +397,8 @@ JsonLinesItemExporter
|
|||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||
``__init__`` method argument to customize this exporter.
|
||||
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -417,8 +411,6 @@ JsonLinesItemExporter
|
|||
Unlike the one produced by :class:`JsonItemExporter`, the format produced by
|
||||
this exporter is well suited for serializing large amounts of data.
|
||||
|
||||
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
|
||||
|
||||
MarshalItemExporter
|
||||
-------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ Here is the code of such extension::
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class SpiderOpenCloseLogging(object):
|
||||
class SpiderOpenCloseLogging:
|
||||
|
||||
def __init__(self, item_count):
|
||||
self.item_count = item_count
|
||||
|
|
@ -364,7 +364,7 @@ Debugger extension
|
|||
|
||||
.. class:: Debugger
|
||||
|
||||
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
|
||||
Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
|
||||
signal is received. After the debugger is exited, the Scrapy process continues
|
||||
running normally.
|
||||
|
||||
|
|
@ -372,5 +372,4 @@ For more info see `Debugging in Python`_.
|
|||
|
||||
This extension only works on POSIX-compliant platforms (i.e. not Windows).
|
||||
|
||||
.. _Python debugger: https://docs.python.org/2/library/pdb.html
|
||||
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export
|
|||
feed") to be consumed by other systems.
|
||||
|
||||
Scrapy provides this functionality out of the box with the Feed Exports, which
|
||||
allows you to generate a feed with the scraped items, using multiple
|
||||
allows you to generate feeds with the scraped items, using multiple
|
||||
serialization formats and storage backends.
|
||||
|
||||
.. _topics-feed-format:
|
||||
|
|
@ -36,7 +36,7 @@ But you can also extend the supported format through the
|
|||
JSON
|
||||
----
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``json``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
|
||||
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
|
||||
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
|
||||
large feeds.
|
||||
|
|
@ -46,7 +46,7 @@ JSON
|
|||
JSON lines
|
||||
----------
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``jsonlines``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
|
||||
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
|
||||
|
||||
.. _topics-feed-format-csv:
|
||||
|
|
@ -54,7 +54,7 @@ JSON lines
|
|||
CSV
|
||||
---
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``csv``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
|
||||
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
|
||||
* To specify columns to export and their order use
|
||||
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
|
||||
|
|
@ -66,7 +66,7 @@ CSV
|
|||
XML
|
||||
---
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``xml``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
|
||||
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
|
||||
|
||||
.. _topics-feed-format-pickle:
|
||||
|
|
@ -74,7 +74,7 @@ XML
|
|||
Pickle
|
||||
------
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``pickle``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
|
||||
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
|
||||
|
||||
.. _topics-feed-format-marshal:
|
||||
|
|
@ -82,7 +82,7 @@ Pickle
|
|||
Marshal
|
||||
-------
|
||||
|
||||
* :setting:`FEED_FORMAT`: ``marshal``
|
||||
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
|
||||
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
|
||||
|
||||
|
|
@ -91,8 +91,8 @@ Marshal
|
|||
Storages
|
||||
========
|
||||
|
||||
When using the feed exports you define where to store the feed using a URI_
|
||||
(through the :setting:`FEED_URI` setting). The feed exports supports multiple
|
||||
When using the feed exports you define where to store the feed using one or multiple URIs_
|
||||
(through the :setting:`FEEDS` setting). The feed exports supports multiple
|
||||
storage backend types which are defined by the URI scheme.
|
||||
|
||||
The storages backends supported out of the box are:
|
||||
|
|
@ -211,41 +211,66 @@ Settings
|
|||
|
||||
These are the settings used for configuring the feed exports:
|
||||
|
||||
* :setting:`FEED_URI` (mandatory)
|
||||
* :setting:`FEED_FORMAT`
|
||||
* :setting:`FEEDS` (mandatory)
|
||||
* :setting:`FEED_EXPORT_ENCODING`
|
||||
* :setting:`FEED_STORE_EMPTY`
|
||||
* :setting:`FEED_EXPORT_FIELDS`
|
||||
* :setting:`FEED_EXPORT_INDENT`
|
||||
* :setting:`FEED_STORAGES`
|
||||
* :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
* :setting:`FEED_EXPORTERS`
|
||||
* :setting:`FEED_STORE_EMPTY`
|
||||
* :setting:`FEED_EXPORT_ENCODING`
|
||||
* :setting:`FEED_EXPORT_FIELDS`
|
||||
* :setting:`FEED_EXPORT_INDENT`
|
||||
|
||||
.. currentmodule:: scrapy.extensions.feedexport
|
||||
|
||||
.. setting:: FEED_URI
|
||||
.. setting:: FEEDS
|
||||
|
||||
FEED_URI
|
||||
--------
|
||||
FEEDS
|
||||
-----
|
||||
|
||||
Default: ``None``
|
||||
.. versionadded:: 2.1
|
||||
|
||||
The URI of the export feed. See :ref:`topics-feed-storage-backends` for
|
||||
supported URI schemes.
|
||||
Default: ``{}``
|
||||
|
||||
This setting is required for enabling the feed exports.
|
||||
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
|
||||
object) and each value is a nested dictionary containing configuration
|
||||
parameters for the specific feed.
|
||||
This setting is required for enabling the feed export feature.
|
||||
|
||||
.. versionchanged:: 2.0
|
||||
Added :class:`pathlib.Path` support.
|
||||
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
||||
|
||||
.. setting:: FEED_FORMAT
|
||||
For instance::
|
||||
|
||||
FEED_FORMAT
|
||||
-----------
|
||||
{
|
||||
'items.json': {
|
||||
'format': 'json',
|
||||
'encoding': 'utf8',
|
||||
'store_empty': False,
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
},
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
'fields': ['name', 'price'],
|
||||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path('items.csv'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
},
|
||||
}
|
||||
|
||||
The serialization format to be used for the feed. See
|
||||
:ref:`topics-feed-format` for possible values.
|
||||
The following is a list of the accepted keys and the setting that is used
|
||||
as a fallback value if that key is not provided for a specific feed definition.
|
||||
|
||||
* ``format``: the serialization format to be used for the feed.
|
||||
See :ref:`topics-feed-format` for possible values.
|
||||
Mandatory, no fallback setting
|
||||
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
|
||||
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
|
||||
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
|
||||
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -400,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
'csv': None,
|
||||
}
|
||||
|
||||
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ contain a price::
|
|||
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class PricePipeline(object):
|
||||
class PricePipeline:
|
||||
|
||||
vat_factor = 1.15
|
||||
|
||||
|
|
@ -103,7 +103,7 @@ format::
|
|||
|
||||
import json
|
||||
|
||||
class JsonWriterPipeline(object):
|
||||
class JsonWriterPipeline:
|
||||
|
||||
def open_spider(self, spider):
|
||||
self.file = open('items.jl', 'w')
|
||||
|
|
@ -132,7 +132,7 @@ method and how to clean up the resources properly.::
|
|||
|
||||
import pymongo
|
||||
|
||||
class MongoPipeline(object):
|
||||
class MongoPipeline:
|
||||
|
||||
collection_name = 'scrapy_items'
|
||||
|
||||
|
|
@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item.
|
|||
from urllib.parse import quote
|
||||
|
||||
|
||||
class ScreenshotPipeline(object):
|
||||
class ScreenshotPipeline:
|
||||
"""Pipeline that uses Splash to render screenshot of
|
||||
every Scrapy item."""
|
||||
|
||||
|
|
@ -219,7 +219,7 @@ returns multiples items with the same id::
|
|||
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class DuplicatesPipeline(object):
|
||||
class DuplicatesPipeline:
|
||||
|
||||
def __init__(self):
|
||||
self.ids_seen = set()
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ especially in a larger project with many spiders.
|
|||
|
||||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
They provide an API similar to :class:`dict` API with a convenient syntax
|
||||
for declaring their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
|
|
@ -24,8 +24,6 @@ serialization can be customized using Item fields metadata, :mod:`trackref`
|
|||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
.. _topics-items-declaring:
|
||||
|
||||
Declaring Items
|
||||
|
|
@ -79,7 +77,7 @@ Working with Items
|
|||
|
||||
Here are some examples of common tasks performed with items, using the
|
||||
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
|
||||
notice the API is very similar to the `dict API`_.
|
||||
notice the API is very similar to the :class:`dict` API.
|
||||
|
||||
Creating items
|
||||
--------------
|
||||
|
|
@ -145,7 +143,7 @@ KeyError: 'Product does not support field: lala'
|
|||
Accessing all populated values
|
||||
------------------------------
|
||||
|
||||
To access all populated values, just use the typical `dict API`_:
|
||||
To access all populated values, just use the typical :class:`dict` API:
|
||||
|
||||
>>> product.keys()
|
||||
['price', 'name']
|
||||
|
|
@ -162,11 +160,9 @@ Copying items
|
|||
To copy an item, you must first decide whether you want a shallow copy or a
|
||||
deep copy.
|
||||
|
||||
If your item contains mutable_ values like lists or dictionaries, a shallow
|
||||
copy will keep references to the same mutable values across all different
|
||||
copies.
|
||||
|
||||
.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
|
||||
If your item contains :term:`mutable` values like lists or dictionaries,
|
||||
a shallow copy will keep references to the same mutable values across all
|
||||
different copies.
|
||||
|
||||
For example, if you have an item with a list of tags, and you create a shallow
|
||||
copy of that item, both the original item and the copy have the same list of
|
||||
|
|
@ -175,9 +171,7 @@ other item as well.
|
|||
|
||||
If that is not the desired behavior, use a deep copy instead.
|
||||
|
||||
See the `documentation of the copy module`_ for more information.
|
||||
|
||||
.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
|
||||
See :mod:`copy` for more information.
|
||||
|
||||
To create a shallow copy of an item, you can either call
|
||||
:meth:`~scrapy.item.Item.copy` on an existing item
|
||||
|
|
@ -235,8 +229,8 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its ``__init__`` method, and
|
||||
also provide the following additional API members:
|
||||
Items replicate the standard :class:`dict` API, including its ``__init__``
|
||||
method, and also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
||||
|
|
@ -249,22 +243,17 @@ Item objects
|
|||
:class:`Field` objects used in the :ref:`Item declaration
|
||||
<topics-items-declaring>`.
|
||||
|
||||
.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
Field objects
|
||||
=============
|
||||
|
||||
.. class:: Field([arg])
|
||||
|
||||
The :class:`Field` class is just an alias to the built-in `dict`_ class and
|
||||
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
|
||||
doesn't provide any extra functionality or attributes. In other words,
|
||||
:class:`Field` objects are plain-old Python dicts. A separate class is used
|
||||
to support the :ref:`item declaration syntax <topics-items-declaring>`
|
||||
based on class attributes.
|
||||
|
||||
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
|
||||
Other classes related to Item
|
||||
=============================
|
||||
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ what is known as a "memory leak".
|
|||
|
||||
To help debugging memory leaks, Scrapy provides a built-in mechanism for
|
||||
tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
|
||||
and you can also use a third-party library called :ref:`Guppy
|
||||
<topics-leaks-guppy>` for more advanced memory debugging (see below for more
|
||||
and you can also use a third-party library called :ref:`muppy
|
||||
<topics-leaks-muppy>` for more advanced memory debugging (see below for more
|
||||
info). Both mechanisms must be used from the :ref:`Telnet Console
|
||||
<topics-telnetconsole>`.
|
||||
|
||||
|
|
@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
.. class:: object_ref
|
||||
|
||||
Inherit from this class (instead of object) if you want to track live
|
||||
Inherit from this class if you want to track live
|
||||
instances with the ``trackref`` module.
|
||||
|
||||
.. function:: print_live_refs(class_name, ignore=NoneType)
|
||||
|
|
@ -193,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
``None`` if none is found. Use :func:`print_live_refs` first to get a list
|
||||
of all tracked live objects per class name.
|
||||
|
||||
.. _topics-leaks-guppy:
|
||||
.. _topics-leaks-muppy:
|
||||
|
||||
Debugging memory leaks with Guppy
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
|
||||
``trackref`` provides a very convenient mechanism for tracking down memory
|
||||
|
|
@ -203,63 +203,9 @@ leaks, but it only keeps track of the objects that are more likely to cause
|
|||
memory leaks (Requests, Responses, Items, and Selectors). However, there are
|
||||
other cases where the memory leaks could come from other (more or less obscure)
|
||||
objects. If this is your case, and you can't find your leaks using ``trackref``,
|
||||
you still have another resource: the `Guppy library`_.
|
||||
If you're using Python3, see :ref:`topics-leaks-muppy`.
|
||||
you still have another resource: the muppy library.
|
||||
|
||||
.. _Guppy library: https://pypi.org/project/guppy/
|
||||
|
||||
If you use ``pip``, you can install Guppy with the following command::
|
||||
|
||||
pip install guppy
|
||||
|
||||
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
|
||||
Guppy heap objects. Here's an example to view all Python objects available in
|
||||
the heap using Guppy:
|
||||
|
||||
>>> x = hpy.heap()
|
||||
>>> x.bytype
|
||||
Partition of a set of 297033 objects. Total size = 52587824 bytes.
|
||||
Index Count % Size % Cumulative % Type
|
||||
0 22307 8 16423880 31 16423880 31 dict
|
||||
1 122285 41 12441544 24 28865424 55 str
|
||||
2 68346 23 5966696 11 34832120 66 tuple
|
||||
3 227 0 5836528 11 40668648 77 unicode
|
||||
4 2461 1 2222272 4 42890920 82 type
|
||||
5 16870 6 2024400 4 44915320 85 function
|
||||
6 13949 5 1673880 3 46589200 89 types.CodeType
|
||||
7 13422 5 1653104 3 48242304 92 list
|
||||
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
|
||||
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
|
||||
<1676 more rows. Type e.g. '_.more' to view.>
|
||||
|
||||
You can see that most space is used by dicts. Then, if you want to see from
|
||||
which attribute those dicts are referenced, you could do:
|
||||
|
||||
>>> x.bytype[0].byvia
|
||||
Partition of a set of 22307 objects. Total size = 16423880 bytes.
|
||||
Index Count % Size % Cumulative % Referred Via:
|
||||
0 10982 49 9416336 57 9416336 57 '.__dict__'
|
||||
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
|
||||
2 3097 14 1122904 7 13220744 80
|
||||
3 990 4 277200 2 13497944 82 "['cookies']"
|
||||
4 987 4 276360 2 13774304 84 "['cache']"
|
||||
5 985 4 275800 2 14050104 86 "['meta']"
|
||||
6 897 4 251160 2 14301264 87 '[2]'
|
||||
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
|
||||
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
|
||||
9 27 0 155016 1 14841328 90 '[1]'
|
||||
<333 more rows. Type e.g. '_.more' to view.>
|
||||
|
||||
As you can see, the Guppy module is very powerful but also requires some deep
|
||||
knowledge about Python internals. For more info about Guppy, refer to the
|
||||
`Guppy documentation`_.
|
||||
|
||||
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
|
||||
|
||||
.. _topics-leaks-muppy:
|
||||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
You can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ Logging
|
|||
explicit calls to the Python standard logging. Keep reading to learn more
|
||||
about the new logging system.
|
||||
|
||||
Scrapy uses `Python's builtin logging system
|
||||
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
|
||||
Scrapy uses :mod:`logging` for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
||||
|
|
@ -83,10 +82,10 @@ path::
|
|||
|
||||
.. seealso::
|
||||
|
||||
Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
|
||||
Module logging, :doc:`HowTo <howto/logging>`
|
||||
Basic Logging Tutorial
|
||||
|
||||
Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
|
||||
Module logging, :ref:`Loggers <logger>`
|
||||
Further documentation on loggers
|
||||
|
||||
.. _topics-logging-from-spiders:
|
||||
|
|
@ -165,14 +164,12 @@ possible levels listed in :ref:`topics-logging-levels`.
|
|||
|
||||
:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
|
||||
used as layouts for all messages. Those strings can contain any placeholders
|
||||
listed in `logging's logrecord attributes docs
|
||||
<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
|
||||
`datetime's strftime and strptime directives
|
||||
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
|
||||
listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
|
||||
:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
|
||||
respectively.
|
||||
|
||||
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
|
||||
component that prints the log. It is unset by default, hence logs contain the
|
||||
component that prints the log. It is unset by default, hence logs contain the
|
||||
Scrapy component responsible for that log output.
|
||||
|
||||
Command-line options
|
||||
|
|
@ -190,7 +187,7 @@ to override some of the Scrapy settings regarding logging.
|
|||
|
||||
.. seealso::
|
||||
|
||||
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
|
||||
Module :mod:`logging.handlers`
|
||||
Further documentation on available handlers
|
||||
|
||||
.. _custom-log-formats:
|
||||
|
|
@ -201,7 +198,7 @@ Custom Log Formats
|
|||
A custom log format can be set for different actions by extending
|
||||
:class:`~scrapy.logformatter.LogFormatter` class and making
|
||||
:setting:`LOG_FORMATTER` point to your new class.
|
||||
|
||||
|
||||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
||||
|
|
@ -256,10 +253,10 @@ scrapy.utils.log module
|
|||
In that case, its usage is not required but it's recommended.
|
||||
|
||||
Another option when running custom scripts is to manually configure the logging.
|
||||
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
|
||||
To do this you can use :func:`logging.basicConfig` to set a basic root handler.
|
||||
|
||||
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
|
||||
so it is recommended to only use `logging.basicConfig()`_ together with
|
||||
so it is recommended to only use :func:`logging.basicConfig` together with
|
||||
:class:`~scrapy.crawler.CrawlerRunner`.
|
||||
|
||||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
|
@ -275,7 +272,3 @@ scrapy.utils.log module
|
|||
|
||||
Refer to :ref:`run-from-script` for more details about using Scrapy this
|
||||
way.
|
||||
|
||||
.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -174,9 +174,9 @@ Request objects
|
|||
See :ref:`topics-request-meta` for a list of special meta keys
|
||||
recognized by Scrapy.
|
||||
|
||||
This dict is `shallow copied`_ when the request is cloned using the
|
||||
``copy()`` or ``replace()`` methods, and can also be accessed, in your
|
||||
spider, from the ``response.meta`` attribute.
|
||||
This dict is :doc:`shallow copied <library/copy>` when the request is
|
||||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.meta`` attribute.
|
||||
|
||||
.. attribute:: Request.cb_kwargs
|
||||
|
||||
|
|
@ -185,11 +185,9 @@ Request objects
|
|||
for new Requests, which means by default callbacks only get a :class:`Response`
|
||||
object as argument.
|
||||
|
||||
This dict is `shallow copied`_ when the request is cloned using the
|
||||
``copy()`` or ``replace()`` methods, and can also be accessed, in your
|
||||
spider, from the ``response.cb_kwargs`` attribute.
|
||||
|
||||
.. _shallow copied: https://docs.python.org/2/library/copy.html
|
||||
This dict is :doc:`shallow copied <library/copy>` when the request is
|
||||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
|
|
@ -566,12 +564,10 @@ dealing with JSON requests.
|
|||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
|
||||
data into JSON format.
|
||||
:type dumps_kwargs: dict
|
||||
|
||||
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
|
||||
|
||||
JsonRequest usage example
|
||||
-------------------------
|
||||
|
||||
|
|
@ -709,7 +705,7 @@ Response objects
|
|||
|
||||
A :class:`twisted.internet.ssl.Certificate` object representing
|
||||
the server's SSL certificate.
|
||||
|
||||
|
||||
Only populated for ``https`` responses, ``None`` otherwise.
|
||||
|
||||
.. attribute:: Response.ip_address
|
||||
|
|
@ -735,18 +731,16 @@ Response objects
|
|||
Constructs an absolute url by combining the Response's :attr:`url` with
|
||||
a possible relative url.
|
||||
|
||||
This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
|
||||
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
||||
making this call::
|
||||
|
||||
urlparse.urljoin(response.url, url)
|
||||
urllib.parse.urljoin(response.url, url)
|
||||
|
||||
.. automethod:: Response.follow
|
||||
|
||||
.. automethod:: Response.follow_all
|
||||
|
||||
|
||||
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
|
||||
|
||||
.. _topics-request-response-ref-response-subclasses:
|
||||
|
||||
Response subclasses
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ achieve this, such as:
|
|||
drawback: it's slow.
|
||||
|
||||
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
|
||||
API based on `ElementTree`_. (lxml is not part of the Python standard
|
||||
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
|
||||
library.)
|
||||
|
||||
Scrapy comes with its own mechanism for extracting data. They're called
|
||||
|
|
@ -36,7 +36,6 @@ defines selectors to associate those styles with specific HTML elements.
|
|||
|
||||
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
|
||||
.. _lxml: https://lxml.de/
|
||||
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
|
||||
.. _XPath: https://www.w3.org/TR/xpath/all/
|
||||
.. _CSS: https://www.w3.org/TR/selectors
|
||||
.. _parsel: https://parsel.readthedocs.io/en/latest/
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
|
|||
|
||||
The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
|
||||
``myproject.settings``. Note that the settings module should be on the
|
||||
Python `import search path`_.
|
||||
|
||||
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
|
||||
Python :ref:`import search path <tut-searchpath>`.
|
||||
|
||||
.. _populating-settings:
|
||||
|
||||
|
|
@ -124,7 +122,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
|
|||
attribute of the Crawler that is passed to ``from_crawler`` method in
|
||||
extensions, middlewares and item pipelines::
|
||||
|
||||
class MyExtension(object):
|
||||
class MyExtension:
|
||||
def __init__(self, log_is_enabled=False):
|
||||
if log_is_enabled:
|
||||
print("log is enabled!")
|
||||
|
|
@ -899,10 +897,9 @@ LOG_FORMAT
|
|||
|
||||
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
|
||||
|
||||
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
|
||||
placeholders.
|
||||
|
||||
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
|
||||
String for formatting log messages. Refer to the
|
||||
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
|
||||
list of available placeholders.
|
||||
|
||||
.. setting:: LOG_DATEFORMAT
|
||||
|
||||
|
|
@ -912,10 +909,9 @@ LOG_DATEFORMAT
|
|||
Default: ``'%Y-%m-%d %H:%M:%S'``
|
||||
|
||||
String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
|
||||
in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
|
||||
directives.
|
||||
|
||||
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
|
||||
in :setting:`LOG_FORMAT`. Refer to the
|
||||
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
|
||||
whole list of available directives.
|
||||
|
||||
.. setting:: LOG_FORMATTER
|
||||
|
||||
|
|
@ -1116,17 +1112,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
|
|||
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
|
||||
this value if you're experiencing problems with insufficient blocking IO.
|
||||
|
||||
.. setting:: REDIRECT_MAX_TIMES
|
||||
|
||||
REDIRECT_MAX_TIMES
|
||||
------------------
|
||||
|
||||
Default: ``20``
|
||||
|
||||
Defines the maximum times a request can be redirected. After this maximum the
|
||||
request's response is returned as is. We used Firefox default value for the
|
||||
same task.
|
||||
|
||||
.. setting:: REDIRECT_PRIORITY_ADJUST
|
||||
|
||||
REDIRECT_PRIORITY_ADJUST
|
||||
|
|
@ -1422,17 +1407,6 @@ Default: ``True``
|
|||
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
|
||||
will be enabled (provided its extension is also enabled).
|
||||
|
||||
.. setting:: TELNETCONSOLE_PORT
|
||||
|
||||
TELNETCONSOLE_PORT
|
||||
------------------
|
||||
|
||||
Default: ``[6023, 6073]``
|
||||
|
||||
The port range to use for the telnet console. If set to ``None`` or ``0``, a
|
||||
dynamically assigned port is used. For more info see
|
||||
:ref:`topics-telnetconsole`.
|
||||
|
||||
.. setting:: TEMPLATES_DIR
|
||||
|
||||
TEMPLATES_DIR
|
||||
|
|
@ -1473,7 +1447,66 @@ If a reactor is already installed,
|
|||
|
||||
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
|
||||
:exc:`Exception` if the installed reactor does not match the
|
||||
:setting:`TWISTED_REACTOR` setting.
|
||||
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
|
||||
:mod:`~twisted.internet.reactor` imports in project files and imported
|
||||
third-party libraries will make Scrapy raise :exc:`Exception` when
|
||||
it checks which reactor is installed.
|
||||
|
||||
In order to use the reactor installed by Scrapy::
|
||||
|
||||
import scrapy
|
||||
from twisted.internet import reactor
|
||||
|
||||
|
||||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop('timeout', '60'))
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
def start_requests(self):
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {'text': quote.css('span.text::text').get()}
|
||||
|
||||
def stop(self):
|
||||
self.crawler.engine.close_spider(self, 'timeout')
|
||||
|
||||
|
||||
which raises :exc:`Exception`, becomes::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class QuotesSpider(scrapy.Spider):
|
||||
name = 'quotes'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.timeout = int(kwargs.pop('timeout', '60'))
|
||||
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||
|
||||
def start_requests(self):
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(self.timeout, self.stop)
|
||||
|
||||
urls = ['http://quotes.toscrape.com/page/1']
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, callback=self.parse)
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {'text': quote.css('span.text::text').get()}
|
||||
|
||||
def stop(self):
|
||||
self.crawler.engine.close_spider(self, 'timeout')
|
||||
|
||||
|
||||
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
|
||||
means that Scrapy will not attempt to install any specific reactor, and the
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param exception: the exception raised
|
||||
:type exception: `Exception`_ object
|
||||
:type exception: :exc:`Exception` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -173,20 +173,16 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
|
||||
If present, this classmethod is called to create a middleware instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the middleware. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for middleware to
|
||||
access them and hook its functionality into Scrapy.
|
||||
|
||||
|
||||
:param crawler: crawler that uses this middleware
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
|
||||
|
||||
|
||||
.. _topics-spider-middleware-ref:
|
||||
|
||||
Built-in spider middleware reference
|
||||
|
|
|
|||
|
|
@ -298,9 +298,7 @@ Keep in mind that spider arguments are only strings.
|
|||
The spider will not do any parsing on its own.
|
||||
If you were to set the ``start_urls`` attribute from the command line,
|
||||
you would have to parse it on your own into a list
|
||||
using something like
|
||||
`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
|
||||
or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
|
||||
using something like :func:`ast.literal_eval` or :func:`json.loads`
|
||||
and then set it as an attribute.
|
||||
Otherwise, you would cause iteration over a ``start_urls`` string
|
||||
(a very common python pitfall)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ Common Stats Collector uses
|
|||
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
|
||||
attribute. Here is an example of an extension that access stats::
|
||||
|
||||
class ExtensionThatAccessStats(object):
|
||||
class ExtensionThatAccessStats:
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
|
|
|||
|
|
@ -40,10 +40,10 @@ the console you need to type::
|
|||
Connected to localhost.
|
||||
Escape character is '^]'.
|
||||
Username:
|
||||
Password:
|
||||
Password:
|
||||
>>>
|
||||
|
||||
By default Username is ``scrapy`` and Password is autogenerated. The
|
||||
By default Username is ``scrapy`` and Password is autogenerated. The
|
||||
autogenerated Password can be seen on Scrapy logs like the example below::
|
||||
|
||||
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
|
||||
|
|
@ -63,7 +63,7 @@ Available variables in the telnet console
|
|||
=========================================
|
||||
|
||||
The telnet console is like a regular Python shell running inside the Scrapy
|
||||
process, so you can do anything from it including importing new modules, etc.
|
||||
process, so you can do anything from it including importing new modules, etc.
|
||||
|
||||
However, the telnet console comes with some default variables defined for
|
||||
convenience:
|
||||
|
|
@ -89,13 +89,11 @@ convenience:
|
|||
+----------------+-------------------------------------------------------------------+
|
||||
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``p`` | a shortcut to the `pprint.pprint`_ function |
|
||||
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
|
||||
.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
|
||||
|
||||
Telnet console usage examples
|
||||
=============================
|
||||
|
||||
|
|
@ -208,4 +206,3 @@ Default: ``None``
|
|||
|
||||
The password used for the telnet console, default behaviour is to have it
|
||||
autogenerated
|
||||
|
||||
|
|
|
|||
|
|
@ -14,40 +14,40 @@ _scrapy() {
|
|||
;;
|
||||
args)
|
||||
case $words[1] in
|
||||
bench)
|
||||
(bench)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
fetch)
|
||||
(fetch)
|
||||
local options=(
|
||||
'--headers[print response HTTP headers instead of body]'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'1::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
genspider)
|
||||
(genspider)
|
||||
local options=(
|
||||
{-l,--list}'[List available templates]'
|
||||
{-e,--edit}'[Edit spider after creating it]'
|
||||
{'(--list)-l','(-l)--list'}'[List available templates]'
|
||||
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
|
||||
'--force[If the spider already exists, overwrite it with the template]'
|
||||
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
|
||||
'1:name:(NAME)'
|
||||
'2:domain:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
runspider)
|
||||
(runspider)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider file:_files -g \*.py'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
settings)
|
||||
(settings)
|
||||
local options=(
|
||||
'--get=[print raw setting value]:option:(SETTING)'
|
||||
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
|
||||
|
|
@ -57,77 +57,77 @@ _scrapy() {
|
|||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
shell)
|
||||
(shell)
|
||||
local options=(
|
||||
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'::file:_files -g \*.html'
|
||||
'::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
startproject)
|
||||
(startproject)
|
||||
local options=(
|
||||
'1:name:(NAME)'
|
||||
'2:dir:_dir_list'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
version)
|
||||
(version)
|
||||
local options=(
|
||||
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
|
||||
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
view)
|
||||
(view)
|
||||
local options=(
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
check)
|
||||
(check)
|
||||
local options=(
|
||||
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
|
||||
{-v,--verbose}'[print contract tests for all spiders]'
|
||||
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
|
||||
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
crawl)
|
||||
(crawl)
|
||||
local options=(
|
||||
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
edit)
|
||||
(edit)
|
||||
local options=(
|
||||
'1:spider:_scrapy_spiders'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
list)
|
||||
(list)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
parse)
|
||||
(parse)
|
||||
local options=(
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
|
||||
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
|
||||
'--pipelines[process items through pipelines]'
|
||||
"--nolinks[don't show links to follow (extracted requests)]"
|
||||
"--noitems[don't show scraped items]"
|
||||
'--nocolour[avoid using pygments to colorize the output]'
|
||||
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
|
||||
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
|
||||
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
|
||||
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
|
||||
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
|
||||
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
|
||||
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
|
||||
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
|
||||
{-v,--verbose}'[print each depth level one by one]'
|
||||
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
|
||||
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
|
|
@ -162,7 +162,7 @@ _scrapy_cmds() {
|
|||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
commands=(${commands[@]} ${project_commands[@]})
|
||||
fi
|
||||
_describe -t common-commands 'common commands' commands
|
||||
_describe -t common-commands 'common commands' commands && ret=0
|
||||
}
|
||||
|
||||
_scrapy_glb_opts() {
|
||||
|
|
@ -172,13 +172,13 @@ _scrapy_glb_opts() {
|
|||
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
|
||||
'--pidfile=[write process ID to FILE]:file:_files'
|
||||
'--profile=[write python cProfile stats to FILE]:file:_files'
|
||||
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
|
||||
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
|
||||
'(-L --loglevel --logfile)--nolog[disable logging completely]'
|
||||
'--pdb[enable pdb on failure]'
|
||||
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
)
|
||||
options=(${options[@]} "$@")
|
||||
_arguments $options
|
||||
_arguments -A "-*" $options && ret=0
|
||||
}
|
||||
|
||||
_httpie_urls() {
|
||||
|
|
|
|||
44
pytest.ini
44
pytest.ini
|
|
@ -36,25 +36,25 @@ flake8-ignore =
|
|||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/genspider.py E128 E501
|
||||
scrapy/commands/parse.py E128 E501 E731
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/shell.py E128 E501
|
||||
scrapy/commands/startproject.py E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128 E127 E502
|
||||
scrapy/core/engine.py E501 E128 E127
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E126
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E241
|
||||
scrapy/core/downloader/middleware.py E501
|
||||
scrapy/core/downloader/tls.py E501
|
||||
scrapy/core/downloader/webclient.py E731 E501 E128 E126
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
|
||||
|
|
@ -97,9 +97,9 @@ flake8-ignore =
|
|||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E501
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
scrapy/pipelines/files.py E116 E501
|
||||
scrapy/pipelines/images.py E501
|
||||
scrapy/pipelines/media.py E125 E501
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
|
|
@ -124,7 +124,7 @@ flake8-ignore =
|
|||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/deprecate.py E128 E501 E127
|
||||
scrapy/utils/gz.py E501 W504
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
|
|
@ -149,14 +149,14 @@ flake8-ignore =
|
|||
scrapy/__init__.py E402 E501
|
||||
scrapy/cmdline.py E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/dupefilters.py E501
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/mail.py E402 E128 E501
|
||||
scrapy/middleware.py E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E501
|
||||
|
|
@ -178,13 +178,13 @@ flake8-ignore =
|
|||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawl.py E501 E741
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
|
|
@ -199,22 +199,22 @@ flake8-ignore =
|
|||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E731 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_feedexport.py E501 F841
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_response.py E501 E128 E265
|
||||
tests/test_http_response.py E501 E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E731 E741 E128 E117 E241
|
||||
tests/test_loader.py E501 E731 E741 E128 E117
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
tests/test_pipeline_files.py E501
|
||||
tests/test_pipeline_images.py F841 E501
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E502
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
|
|
@ -226,7 +226,7 @@ flake8-ignore =
|
|||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
|
|
@ -235,7 +235,7 @@ flake8-ignore =
|
|||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E241
|
||||
tests/test_utils_iterators.py E501 E128 E129
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501 E731
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
|
|
@ -243,8 +243,8 @@ flake8-ignore =
|
|||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E731
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E123 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError
|
|||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.utils.project import inside_project, get_project_settings
|
||||
from scrapy.utils.python import garbage_collect
|
||||
from scrapy.settings.deprecated import check_deprecated_settings
|
||||
|
||||
|
||||
def _iter_command_classes(module_name):
|
||||
|
|
@ -118,7 +117,6 @@ def execute(argv=None, settings=None):
|
|||
pass
|
||||
else:
|
||||
settings['EDITOR'] = editor
|
||||
check_deprecated_settings(settings)
|
||||
|
||||
inproject = inside_project()
|
||||
cmds = _get_commands_dict(settings, inproject)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict
|
|||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
class ScrapyCommand(object):
|
||||
class ScrapyCommand:
|
||||
|
||||
requires_project = False
|
||||
crawler_process = None
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class Command(ScrapyCommand):
|
|||
self.crawler_process.start()
|
||||
|
||||
|
||||
class _BenchServer(object):
|
||||
class _BenchServer:
|
||||
|
||||
def __enter__(self):
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import os
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.python import without_none_values
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
|
|
@ -19,7 +17,7 @@ class Command(ScrapyCommand):
|
|||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE",
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
|
|
@ -31,21 +29,8 @@ class Command(ScrapyCommand):
|
|||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
if opts.output == '-':
|
||||
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
|
||||
else:
|
||||
self.settings.set('FEED_URI', opts.output, priority='cmdline')
|
||||
feed_exporters = without_none_values(
|
||||
self.settings.getwithbase('FEED_EXPORTERS'))
|
||||
valid_output_formats = feed_exporters.keys()
|
||||
if not opts.output_format:
|
||||
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
|
||||
if opts.output_format not in valid_output_formats:
|
||||
raise UsageError("Unrecognized output format '%s', set one"
|
||||
" using the '-t' switch or as a file extension"
|
||||
" from the supported list %s" % (opts.output_format,
|
||||
tuple(valid_output_formats)))
|
||||
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) < 1:
|
||||
|
|
|
|||
|
|
@ -90,8 +90,7 @@ class Command(ScrapyCommand):
|
|||
'module': module,
|
||||
'name': name,
|
||||
'domain': domain,
|
||||
'classname': '%sSpider' % ''.join(s.capitalize() \
|
||||
for s in module.split('_'))
|
||||
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
|
||||
}
|
||||
if self.settings.get('NEWSPIDER_MODULE'):
|
||||
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
|
||||
|
|
@ -102,8 +101,8 @@ class Command(ScrapyCommand):
|
|||
spider_file = "%s.py" % join(spiders_dir, module)
|
||||
shutil.copyfile(template_file, spider_file)
|
||||
render_templatefile(spider_file, **tvars)
|
||||
print("Created spider %r using template %r " % (name, \
|
||||
template_name), end=('' if spiders_module else '\n'))
|
||||
print("Created spider %r using template %r "
|
||||
% (name, template_name), end=('' if spiders_module else '\n'))
|
||||
if spiders_module:
|
||||
print("in module:\n %s.%s" % (spiders_module.__name__, module))
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,7 @@ from importlib import import_module
|
|||
from scrapy.utils.spider import iter_spider_classes
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.python import without_none_values
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
|
||||
|
||||
def _import_file(filepath):
|
||||
|
|
@ -43,7 +42,7 @@ class Command(ScrapyCommand):
|
|||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE",
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
|
|
@ -55,19 +54,8 @@ class Command(ScrapyCommand):
|
|||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
if opts.output == '-':
|
||||
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
|
||||
else:
|
||||
self.settings.set('FEED_URI', opts.output, priority='cmdline')
|
||||
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
|
||||
if not opts.output_format:
|
||||
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
|
||||
if opts.output_format not in feed_exporters:
|
||||
raise UsageError("Unrecognized output format '%s', set one"
|
||||
" using the '-t' switch or as a file extension"
|
||||
" from the supported list %s" % (opts.output_format,
|
||||
tuple(feed_exporters)))
|
||||
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1:
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class Command(ScrapyCommand):
|
|||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output
|
|||
from scrapy.utils.python import get_spec
|
||||
|
||||
|
||||
class ContractsManager(object):
|
||||
class ContractsManager:
|
||||
contracts = {}
|
||||
|
||||
def __init__(self, contracts):
|
||||
|
|
@ -107,7 +107,7 @@ class ContractsManager(object):
|
|||
request.errback = eb_wrapper
|
||||
|
||||
|
||||
class Contract(object):
|
||||
class Contract:
|
||||
""" Abstract class for contracts """
|
||||
request_cls = None
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from time import time
|
|||
from datetime import datetime
|
||||
from collections import deque
|
||||
|
||||
from twisted.internet import reactor, defer, task
|
||||
from twisted.internet import defer, task
|
||||
|
||||
from scrapy.utils.defer import mustbe_deferred
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
|
|||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
|
||||
|
||||
class Slot(object):
|
||||
class Slot:
|
||||
"""Downloader slot"""
|
||||
|
||||
def __init__(self, concurrency, delay, randomize_delay):
|
||||
|
|
@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
|
|||
return concurrency, delay
|
||||
|
||||
|
||||
class Downloader(object):
|
||||
class Downloader:
|
||||
|
||||
DOWNLOAD_SLOT = 'download_slot'
|
||||
|
||||
|
|
@ -133,6 +133,7 @@ class Downloader(object):
|
|||
return deferred
|
||||
|
||||
def _process_queue(self, spider, slot):
|
||||
from twisted.internet import reactor
|
||||
if slot.latercall and slot.latercall.active():
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ import re
|
|||
from io import BytesIO
|
||||
from urllib.parse import unquote
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet.protocol import ClientCreator, Protocol
|
||||
from twisted.protocols.ftp import CommandFailed, FTPClient
|
||||
|
||||
|
|
@ -81,6 +80,7 @@ class FTPDownloadHandler:
|
|||
return cls(crawler.settings)
|
||||
|
||||
def download_request(self, request, spider):
|
||||
from twisted.internet import reactor
|
||||
parsed_url = urlparse_cached(request)
|
||||
user = request.meta.get("ftp_user", self.default_user)
|
||||
password = request.meta.get("ftp_password", self.default_password)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
"""Download handlers for http and https schemes
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
|
@ -26,6 +24,7 @@ class HTTP10DownloadHandler:
|
|||
return factory.deferred
|
||||
|
||||
def _connect(self, factory):
|
||||
from twisted.internet import reactor
|
||||
host, port = to_unicode(factory.host), factory.port
|
||||
if factory.scheme == b'https':
|
||||
client_context_factory = create_instance(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from io import BytesIO
|
|||
from time import time
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
from twisted.internet import defer, protocol, reactor, ssl
|
||||
from twisted.internet import defer, protocol, ssl
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
|
||||
|
|
@ -34,6 +34,7 @@ class HTTP11DownloadHandler:
|
|||
lazy = False
|
||||
|
||||
def __init__(self, settings, crawler=None):
|
||||
from twisted.internet import reactor
|
||||
self._pool = HTTPConnectionPool(reactor, persistent=True)
|
||||
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
|
||||
self._pool._factory.noisy = False
|
||||
|
|
@ -82,6 +83,7 @@ class HTTP11DownloadHandler:
|
|||
return agent.download_request(request)
|
||||
|
||||
def close(self):
|
||||
from twisted.internet import reactor
|
||||
d = self._pool.closeCachedConnections()
|
||||
# closeCachedConnections will hang on network or server issues, so
|
||||
# we'll manually timeout the deferred.
|
||||
|
|
@ -267,7 +269,7 @@ class ScrapyProxyAgent(Agent):
|
|||
)
|
||||
|
||||
|
||||
class ScrapyAgent(object):
|
||||
class ScrapyAgent:
|
||||
|
||||
_Agent = Agent
|
||||
_ProxyAgent = ScrapyProxyAgent
|
||||
|
|
@ -285,6 +287,7 @@ class ScrapyAgent(object):
|
|||
self._txresponse = None
|
||||
|
||||
def _get_agent(self, request, timeout):
|
||||
from twisted.internet import reactor
|
||||
bindaddress = request.meta.get('bindaddress') or self._bindAddress
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
|
|
@ -327,6 +330,7 @@ class ScrapyAgent(object):
|
|||
)
|
||||
|
||||
def download_request(self, request):
|
||||
from twisted.internet import reactor
|
||||
timeout = request.meta.get('download_timeout') or self._connectTimeout
|
||||
agent = self._get_agent(request, timeout)
|
||||
|
||||
|
|
@ -338,20 +342,6 @@ class ScrapyAgent(object):
|
|||
headers.removeHeader(b'Proxy-Authorization')
|
||||
if request.body:
|
||||
bodyproducer = _RequestBodyProducer(request.body)
|
||||
elif method == b'POST':
|
||||
# Setting Content-Length: 0 even for POST requests is not a
|
||||
# MUST per HTTP RFCs, but it's common behavior, and some
|
||||
# servers require this, otherwise returning HTTP 411 Length required
|
||||
#
|
||||
# RFC 7230#section-3.3.2:
|
||||
# "a Content-Length header field is normally sent in a POST
|
||||
# request even when the value is 0 (indicating an empty payload body)."
|
||||
#
|
||||
# Twisted < 17 will not add "Content-Length: 0" by itself;
|
||||
# Twisted >= 17 fixes this;
|
||||
# Using a producer with an empty-string sends `0` as Content-Length
|
||||
# for all versions of Twisted.
|
||||
bodyproducer = _RequestBodyProducer(b'')
|
||||
else:
|
||||
bodyproducer = None
|
||||
start_time = time()
|
||||
|
|
@ -440,7 +430,7 @@ class ScrapyAgent(object):
|
|||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
class _RequestBodyProducer(object):
|
||||
class _RequestBodyProducer:
|
||||
|
||||
def __init__(self, body):
|
||||
self.body = body
|
||||
|
|
|
|||
|
|
@ -35,38 +35,44 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
for method in self.methods['process_request']:
|
||||
response = yield deferred_from_coro(method(request=request, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
|
||||
(method.__self__.__class__.__name__, response.__class__.__name__))
|
||||
raise _InvalidOutput(
|
||||
"Middleware %s.process_request must return None, Response or Request, got %s"
|
||||
% (method.__self__.__class__.__name__, response.__class__.__name__)
|
||||
)
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue((yield download_func(request=request, spider=spider)))
|
||||
return response
|
||||
return (yield download_func(request=request, spider=spider))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_response(response):
|
||||
assert response is not None, 'Received None in process_response'
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
if not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
raise _InvalidOutput(
|
||||
"Middleware %s.process_response must return Response or Request, got %s"
|
||||
% (method.__self__.__class__.__name__, type(response))
|
||||
)
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(response)
|
||||
return response
|
||||
return response
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_exception(_failure):
|
||||
exception = _failure.value
|
||||
def process_exception(failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
|
||||
(method.__self__.__class__.__name__, type(response)))
|
||||
raise _InvalidOutput(
|
||||
"Middleware %s.process_exception must return None, Response or Request, got %s"
|
||||
% (method.__self__.__class__.__name__, type(response))
|
||||
)
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(_failure)
|
||||
return response
|
||||
return failure
|
||||
|
||||
deferred = mustbe_deferred(process_request, request)
|
||||
deferred.addErrback(process_exception)
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ METHOD_TLSv12 = 'TLSv1.2'
|
|||
|
||||
|
||||
openssl_methods = {
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
|
||||
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
|
||||
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
|
||||
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Slot(object):
|
||||
class Slot:
|
||||
|
||||
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
|
||||
self.closing = False
|
||||
|
|
@ -53,7 +53,7 @@ class Slot(object):
|
|||
self.closing.callback(None)
|
||||
|
||||
|
||||
class ExecutionEngine(object):
|
||||
class ExecutionEngine:
|
||||
|
||||
def __init__(self, crawler, spider_closed_callback):
|
||||
self.crawler = crawler
|
||||
|
|
@ -277,10 +277,9 @@ class ExecutionEngine(object):
|
|||
next loop and this function is guaranteed to be called (at least) once
|
||||
again for this spider.
|
||||
"""
|
||||
res = self.signals.send_catch_log(signal=signals.spider_idle, \
|
||||
res = self.signals.send_catch_log(signal=signals.spider_idle,
|
||||
spider=spider, dont_log=DontCloseSpider)
|
||||
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
|
||||
for _, x in res):
|
||||
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
|
||||
return
|
||||
|
||||
if self.spider_is_idle(spider):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Scheduler(object):
|
||||
class Scheduler:
|
||||
"""
|
||||
Scrapy Scheduler. It allows to enqueue requests and then get
|
||||
a next request to download. Scheduler is also handling duplication
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Slot(object):
|
||||
class Slot:
|
||||
"""Scraper slot (one per running spider)"""
|
||||
|
||||
MIN_RESPONSE_SIZE = 1024
|
||||
|
|
@ -62,7 +62,7 @@ class Slot(object):
|
|||
return self.active_size > self.max_active_size
|
||||
|
||||
|
||||
class Scraper(object):
|
||||
class Scraper:
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.slot = None
|
||||
|
|
|
|||
|
|
@ -4,7 +4,15 @@ import signal
|
|||
import warnings
|
||||
|
||||
from twisted.internet import defer
|
||||
from zope.interface.verify import DoesNotImplement, verifyClass
|
||||
from zope.interface.exceptions import DoesNotImplement
|
||||
|
||||
try:
|
||||
# zope >= 5.0 only supports MultipleInvalid
|
||||
from zope.interface.exceptions import MultipleInvalid
|
||||
except ImportError:
|
||||
MultipleInvalid = None
|
||||
|
||||
from zope.interface.verify import verifyClass
|
||||
|
||||
from scrapy import signals, Spider
|
||||
from scrapy.core.engine import ExecutionEngine
|
||||
|
|
@ -68,17 +76,6 @@ class Crawler:
|
|||
self.spider = None
|
||||
self.engine = None
|
||||
|
||||
@property
|
||||
def spiders(self):
|
||||
if not hasattr(self, '_spiders'):
|
||||
warnings.warn("Crawler.spiders is deprecated, use "
|
||||
"CrawlerRunner.spider_loader or instantiate "
|
||||
"scrapy.spiderloader.SpiderLoader with your "
|
||||
"settings.",
|
||||
category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._spiders = _get_spider_loader(self.settings.frozencopy())
|
||||
return self._spiders
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def crawl(self, *args, **kwargs):
|
||||
assert not self.crawling, "Crawling already taking place"
|
||||
|
|
@ -130,11 +127,28 @@ class CrawlerRunner:
|
|||
":meth:`crawl` and managed by this class."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_spider_loader(settings):
|
||||
""" Get SpiderLoader instance from settings """
|
||||
cls_path = settings.get('SPIDER_LOADER_CLASS')
|
||||
loader_cls = load_object(cls_path)
|
||||
excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
|
||||
try:
|
||||
verifyClass(ISpiderLoader, loader_cls)
|
||||
except excs:
|
||||
warnings.warn(
|
||||
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
|
||||
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
|
||||
'Please add all missing methods to avoid unexpected runtime errors.',
|
||||
category=ScrapyDeprecationWarning, stacklevel=2
|
||||
)
|
||||
return loader_cls.from_settings(settings.frozencopy())
|
||||
|
||||
def __init__(self, settings=None):
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
self.settings = settings
|
||||
self.spider_loader = _get_spider_loader(settings)
|
||||
self.spider_loader = self._get_spider_loader(settings)
|
||||
self._crawlers = set()
|
||||
self._active = set()
|
||||
self.bootstrap_failed = False
|
||||
|
|
@ -327,19 +341,3 @@ class CrawlerProcess(CrawlerRunner):
|
|||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
||||
|
||||
def _get_spider_loader(settings):
|
||||
""" Get SpiderLoader instance from settings """
|
||||
cls_path = settings.get('SPIDER_LOADER_CLASS')
|
||||
loader_cls = load_object(cls_path)
|
||||
try:
|
||||
verifyClass(ISpiderLoader, loader_cls)
|
||||
except DoesNotImplement:
|
||||
warnings.warn(
|
||||
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
|
||||
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
|
||||
'Please add all missing methods to avoid unexpected runtime errors.',
|
||||
category=ScrapyDeprecationWarning, stacklevel=2
|
||||
)
|
||||
return loader_cls.from_settings(settings.frozencopy())
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AjaxCrawlMiddleware(object):
|
||||
class AjaxCrawlMiddleware:
|
||||
"""
|
||||
Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
|
||||
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.http import decode_chunked_transfer
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
|
||||
"chunked transfers are supported by default.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
|
||||
class ChunkedTransferMiddleware(object):
|
||||
"""This middleware adds support for chunked transfer encoding, as
|
||||
documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if response.headers.get('Transfer-Encoding') == 'chunked':
|
||||
body = decode_chunked_transfer(response.body)
|
||||
return response.replace(body=body)
|
||||
return response
|
||||
|
|
@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CookiesMiddleware(object):
|
||||
class CookiesMiddleware:
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
|
||||
def __init__(self, debug=False):
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DecompressionMiddleware(object):
|
||||
class DecompressionMiddleware:
|
||||
""" This middleware tries to recognise and extract the possibly compressed
|
||||
responses that may arrive. """
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
|
|||
from scrapy.utils.python import without_none_values
|
||||
|
||||
|
||||
class DefaultHeadersMiddleware(object):
|
||||
class DefaultHeadersMiddleware:
|
||||
|
||||
def __init__(self, headers):
|
||||
self._headers = headers
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
|
|||
from scrapy import signals
|
||||
|
||||
|
||||
class DownloadTimeoutMiddleware(object):
|
||||
class DownloadTimeoutMiddleware:
|
||||
|
||||
def __init__(self, timeout=180):
|
||||
self._timeout = timeout
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header
|
|||
from scrapy import signals
|
||||
|
||||
|
||||
class HttpAuthMiddleware(object):
|
||||
class HttpAuthMiddleware:
|
||||
"""Set Basic HTTP Authorization header
|
||||
(http_user and http_pass spider class attributes)"""
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
|
|||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class HttpCacheMiddleware(object):
|
||||
class HttpCacheMiddleware:
|
||||
|
||||
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ except ImportError:
|
|||
pass
|
||||
|
||||
|
||||
class HttpCompressionMiddleware(object):
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites"""
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
from scrapy.utils.python import to_bytes
|
||||
|
||||
|
||||
class HttpProxyMiddleware(object):
|
||||
class HttpProxyMiddleware:
|
||||
|
||||
def __init__(self, auth_encoding='latin-1'):
|
||||
self.auth_encoding = auth_encoding
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseRedirectMiddleware(object):
|
||||
class BaseRedirectMiddleware:
|
||||
|
||||
enabled_setting = 'REDIRECT_ENABLED'
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RetryMiddleware(object):
|
||||
class RetryMiddleware:
|
||||
|
||||
# IOError is raised by the HttpCompression middleware when trying to
|
||||
# decompress an empty response
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RobotsTxtMiddleware(object):
|
||||
class RobotsTxtMiddleware:
|
||||
DOWNLOAD_PRIORITY = 1000
|
||||
|
||||
def __init__(self, crawler):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr
|
|||
from scrapy.utils.python import global_object_name
|
||||
|
||||
|
||||
class DownloaderStats(object):
|
||||
class DownloaderStats:
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from scrapy import signals
|
||||
|
||||
|
||||
class UserAgentMiddleware(object):
|
||||
class UserAgentMiddleware:
|
||||
"""This middleware allows spiders to override the user_agent"""
|
||||
|
||||
def __init__(self, user_agent='Scrapy'):
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir
|
|||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
|
||||
|
||||
class BaseDupeFilter(object):
|
||||
class BaseDupeFilter:
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
|
|
@ -61,7 +61,7 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
def log(self, request, spider):
|
||||
if self.debug:
|
||||
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
|
||||
args = {'request': request, 'referer': referer_str(request) }
|
||||
args = {'request': request, 'referer': referer_str(request)}
|
||||
self.logger.debug(msg, args, extra={'spider': spider})
|
||||
elif self.logdupes:
|
||||
msg = ("Filtered duplicate request: %(request)s"
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
|
|||
'JsonItemExporter', 'MarshalItemExporter']
|
||||
|
||||
|
||||
class BaseItemExporter(object):
|
||||
class BaseItemExporter:
|
||||
|
||||
def __init__(self, *, dont_fail=False, **kwargs):
|
||||
self._kwargs = kwargs
|
||||
|
|
|
|||
|
|
@ -6,13 +6,11 @@ See documentation in docs/topics/extensions.rst
|
|||
|
||||
from collections import defaultdict
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
||||
class CloseSpider(object):
|
||||
class CloseSpider:
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.crawler = crawler
|
||||
|
|
@ -54,6 +52,7 @@ class CloseSpider(object):
|
|||
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
|
||||
|
||||
def spider_opened(self, spider):
|
||||
from twisted.internet import reactor
|
||||
self.task = reactor.callLater(self.close_on['timeout'],
|
||||
self.crawler.engine.close_spider, spider,
|
||||
reason='closespider_timeout')
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from datetime import datetime
|
|||
from scrapy import signals
|
||||
|
||||
|
||||
class CoreStats(object):
|
||||
class CoreStats:
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StackTraceDump(object):
|
||||
class StackTraceDump:
|
||||
|
||||
def __init__(self, crawler=None):
|
||||
self.crawler = crawler
|
||||
|
|
@ -52,7 +52,7 @@ class StackTraceDump(object):
|
|||
return dumps
|
||||
|
||||
|
||||
class Debugger(object):
|
||||
class Debugger:
|
||||
def __init__(self):
|
||||
try:
|
||||
signal.signal(signal.SIGUSR2, self._enter_debugger)
|
||||
|
|
|
|||
|
|
@ -4,24 +4,27 @@ Feed Exports extension
|
|||
See documentation in docs/topics/feed-exports.rst
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from tempfile import NamedTemporaryFile
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse, unquote
|
||||
from tempfile import NamedTemporaryFile
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
from zope.interface import Interface, implementer
|
||||
from twisted.internet import defer, threads
|
||||
from w3lib.url import file_uri_to_path
|
||||
from zope.interface import implementer, Interface
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import without_none_values
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.utils.boto import is_botocore
|
||||
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.python import without_none_values
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -41,7 +44,7 @@ class IFeedStorage(Interface):
|
|||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class BlockingFeedStorage(object):
|
||||
class BlockingFeedStorage:
|
||||
|
||||
def open(self, spider):
|
||||
path = spider.crawler.settings['FEED_TEMPDIR']
|
||||
|
|
@ -58,7 +61,7 @@ class BlockingFeedStorage(object):
|
|||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class StdoutFeedStorage(object):
|
||||
class StdoutFeedStorage:
|
||||
|
||||
def __init__(self, uri, _stdout=None):
|
||||
if not _stdout:
|
||||
|
|
@ -73,7 +76,7 @@ class StdoutFeedStorage(object):
|
|||
|
||||
|
||||
@implementer(IFeedStorage)
|
||||
class FileFeedStorage(object):
|
||||
class FileFeedStorage:
|
||||
|
||||
def __init__(self, uri):
|
||||
self.path = file_uri_to_path(uri)
|
||||
|
|
@ -98,8 +101,6 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
from scrapy.utils.project import get_project_settings
|
||||
settings = get_project_settings()
|
||||
if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
|
||||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn(
|
||||
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
|
||||
"without AWS keys is deprecated. Please supply credentials or "
|
||||
|
|
@ -178,88 +179,117 @@ class FTPFeedStorage(BlockingFeedStorage):
|
|||
)
|
||||
|
||||
|
||||
class SpiderSlot(object):
|
||||
def __init__(self, file, exporter, storage, uri):
|
||||
class _FeedSlot:
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty):
|
||||
self.file = file
|
||||
self.exporter = exporter
|
||||
self.storage = storage
|
||||
# feed params
|
||||
self.uri = uri
|
||||
self.format = format
|
||||
self.store_empty = store_empty
|
||||
# flags
|
||||
self.itemcount = 0
|
||||
|
||||
|
||||
class FeedExporter(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
if not settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
self.urifmt = str(settings['FEED_URI'])
|
||||
self.format = settings['FEED_FORMAT'].lower()
|
||||
self.export_encoding = settings['FEED_EXPORT_ENCODING']
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
self.exporters = self._load_components('FEED_EXPORTERS')
|
||||
if not self._storage_supported(self.urifmt):
|
||||
raise NotConfigured
|
||||
if not self._exporter_supported(self.format):
|
||||
raise NotConfigured
|
||||
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
|
||||
self._exporting = False
|
||||
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
|
||||
self.indent = None
|
||||
if settings.get('FEED_EXPORT_INDENT') is not None:
|
||||
self.indent = settings.getint('FEED_EXPORT_INDENT')
|
||||
uripar = settings['FEED_URI_PARAMS']
|
||||
self._uripar = load_object(uripar) if uripar else lambda x, y: None
|
||||
|
||||
def start_exporting(self):
|
||||
if not self._exporting:
|
||||
self.exporter.start_exporting()
|
||||
self._exporting = True
|
||||
|
||||
def finish_exporting(self):
|
||||
if self._exporting:
|
||||
self.exporter.finish_exporting()
|
||||
self._exporting = False
|
||||
|
||||
|
||||
class FeedExporter:
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
o = cls(crawler.settings)
|
||||
o.crawler = crawler
|
||||
crawler.signals.connect(o.open_spider, signals.spider_opened)
|
||||
crawler.signals.connect(o.close_spider, signals.spider_closed)
|
||||
crawler.signals.connect(o.item_scraped, signals.item_scraped)
|
||||
return o
|
||||
exporter = cls(crawler)
|
||||
crawler.signals.connect(exporter.open_spider, signals.spider_opened)
|
||||
crawler.signals.connect(exporter.close_spider, signals.spider_closed)
|
||||
crawler.signals.connect(exporter.item_scraped, signals.item_scraped)
|
||||
return exporter
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.crawler = crawler
|
||||
self.settings = crawler.settings
|
||||
self.feeds = {}
|
||||
self.slots = []
|
||||
|
||||
if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
|
||||
# Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings
|
||||
if self.settings['FEED_URI']:
|
||||
warnings.warn(
|
||||
'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of '
|
||||
'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details',
|
||||
category=ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
|
||||
feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
|
||||
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
|
||||
|
||||
# 'FEEDS' setting takes precedence over 'FEED_URI'
|
||||
for uri, feed in self.settings.getdict('FEEDS').items():
|
||||
uri = str(uri) # handle pathlib.Path objects
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
|
||||
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
self.exporters = self._load_components('FEED_EXPORTERS')
|
||||
for uri, feed in self.feeds.items():
|
||||
if not self._storage_supported(uri):
|
||||
raise NotConfigured
|
||||
if not self._exporter_supported(feed['format']):
|
||||
raise NotConfigured
|
||||
|
||||
def open_spider(self, spider):
|
||||
uri = self.urifmt % self._get_uri_params(spider)
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(file, fields_to_export=self.export_fields,
|
||||
encoding=self.export_encoding, indent=self.indent)
|
||||
if self.store_empty:
|
||||
exporter.start_exporting()
|
||||
self._exporting = True
|
||||
self.slot = SpiderSlot(file, exporter, storage, uri)
|
||||
for uri, feed in self.feeds.items():
|
||||
uri = uri % self._get_uri_params(spider, feed['uri_params'])
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
)
|
||||
slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
|
||||
self.slots.append(slot)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
|
||||
def close_spider(self, spider):
|
||||
slot = self.slot
|
||||
if not slot.itemcount and not self.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
if self._exporting:
|
||||
slot.exporter.finish_exporting()
|
||||
self._exporting = False
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': self.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
|
||||
extra={'spider': spider}))
|
||||
d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
return d
|
||||
deferred_list = []
|
||||
for slot in self.slots:
|
||||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
|
||||
extra={'spider': spider}))
|
||||
d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
deferred_list.append(d)
|
||||
return defer.DeferredList(deferred_list) if deferred_list else None
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
slot = self.slot
|
||||
if not self._exporting:
|
||||
slot.exporter.start_exporting()
|
||||
self._exporting = True
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
return item
|
||||
for slot in self.slots:
|
||||
slot.start_exporting()
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
|
||||
def _load_components(self, setting_prefix):
|
||||
conf = without_none_values(self.settings.getwithbase(setting_prefix))
|
||||
|
|
@ -295,17 +325,18 @@ class FeedExporter(object):
|
|||
objcls, self.settings, getattr(self, 'crawler', None),
|
||||
*args, **kwargs)
|
||||
|
||||
def _get_exporter(self, *args, **kwargs):
|
||||
return self._get_instance(self.exporters[self.format], *args, **kwargs)
|
||||
def _get_exporter(self, file, format, *args, **kwargs):
|
||||
return self._get_instance(self.exporters[format], file, *args, **kwargs)
|
||||
|
||||
def _get_storage(self, uri):
|
||||
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
|
||||
|
||||
def _get_uri_params(self, spider):
|
||||
def _get_uri_params(self, spider, uri_params):
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['time'] = ts
|
||||
self._uripar(params, spider)
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPolicy(object):
|
||||
class DummyPolicy:
|
||||
|
||||
def __init__(self, settings):
|
||||
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
|
||||
|
|
@ -39,7 +39,7 @@ class DummyPolicy(object):
|
|||
return True
|
||||
|
||||
|
||||
class RFC2616Policy(object):
|
||||
class RFC2616Policy:
|
||||
|
||||
MAXAGE = 3600 * 24 * 365 # one year
|
||||
|
||||
|
|
@ -213,7 +213,7 @@ class RFC2616Policy(object):
|
|||
return currentage
|
||||
|
||||
|
||||
class DbmCacheStorage(object):
|
||||
class DbmCacheStorage:
|
||||
|
||||
def __init__(self, settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
|
||||
|
|
@ -270,7 +270,7 @@ class DbmCacheStorage(object):
|
|||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class FilesystemCacheStorage(object):
|
||||
class FilesystemCacheStorage:
|
||||
|
||||
def __init__(self, settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from scrapy import signals
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LogStats(object):
|
||||
class LogStats:
|
||||
"""Log basic scraping stats periodically"""
|
||||
|
||||
def __init__(self, stats, interval=60.0):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured
|
|||
from scrapy.utils.trackref import live_refs
|
||||
|
||||
|
||||
class MemoryDebugger(object):
|
||||
class MemoryDebugger:
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryUsage(object):
|
||||
class MemoryUsage:
|
||||
|
||||
def __init__(self, crawler):
|
||||
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured
|
|||
from scrapy.utils.job import job_dir
|
||||
|
||||
|
||||
class SpiderState(object):
|
||||
class SpiderState:
|
||||
"""Store and load spider state during a scraping job"""
|
||||
|
||||
def __init__(self, jobdir=None):
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from scrapy import signals
|
|||
from scrapy.mail import MailSender
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
class StatsMailer(object):
|
||||
class StatsMailer:
|
||||
|
||||
def __init__(self, stats, recipients, mail):
|
||||
self.stats = stats
|
||||
|
|
|
|||
|
|
@ -26,11 +26,6 @@ from scrapy.utils.engine import print_engine_status
|
|||
from scrapy.utils.reactor import listen_tcp
|
||||
from scrapy.utils.decorators import defers
|
||||
|
||||
try:
|
||||
import guppy
|
||||
hpy = guppy.hpy()
|
||||
except ImportError:
|
||||
hpy = None
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -110,7 +105,6 @@ class TelnetConsole(protocol.ServerFactory):
|
|||
'est': lambda: print_engine_status(self.crawler.engine),
|
||||
'p': pprint.pprint,
|
||||
'prefs': print_live_refs,
|
||||
'hpy': hpy,
|
||||
'help': "This is Scrapy telnet console. For more info see: "
|
||||
"https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from scrapy import signals
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AutoThrottle(object):
|
||||
class AutoThrottle:
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.crawler = crawler
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
from scrapy.utils.python import to_unicode
|
||||
|
||||
|
||||
class CookieJar(object):
|
||||
class CookieJar:
|
||||
def __init__(self, policy=None, check_expired_frequency=10000):
|
||||
self.policy = policy or DefaultCookiePolicy()
|
||||
self.jar = _CookieJar(self.policy)
|
||||
|
|
@ -100,7 +100,7 @@ def potential_domain_matches(domain):
|
|||
return matches + ['.' + d for d in matches]
|
||||
|
||||
|
||||
class _DummyLock(object):
|
||||
class _DummyLock:
|
||||
def acquire(self):
|
||||
pass
|
||||
|
||||
|
|
@ -108,7 +108,7 @@ class _DummyLock(object):
|
|||
pass
|
||||
|
||||
|
||||
class WrappedRequest(object):
|
||||
class WrappedRequest:
|
||||
"""Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
|
||||
|
||||
see http://docs.python.org/library/urllib2.html#urllib2.Request
|
||||
|
|
@ -178,7 +178,7 @@ class WrappedRequest(object):
|
|||
self.request.headers.appendlist(name, value)
|
||||
|
||||
|
||||
class WrappedResponse(object):
|
||||
class WrappedResponse:
|
||||
|
||||
def __init__(self, response):
|
||||
self.response = response
|
||||
|
|
|
|||
|
|
@ -129,6 +129,9 @@ class Request(object_ref):
|
|||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
|
||||
may modify the :class:`~scrapy.http.Request` object.
|
||||
|
||||
To translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
"""
|
||||
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
|
||||
request_kwargs.update(kwargs)
|
||||
|
|
|
|||
|
|
@ -188,9 +188,11 @@ class TextResponse(Response):
|
|||
selectors from which links cannot be obtained (for instance, anchor tags without an
|
||||
``href`` attribute)
|
||||
"""
|
||||
arg_count = len(list(filter(None, (urls, css, xpath))))
|
||||
if arg_count != 1:
|
||||
raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
|
||||
arguments = [x for x in (urls, css, xpath) if x is not None]
|
||||
if len(arguments) != 1:
|
||||
raise ValueError(
|
||||
"Please supply exactly one of the following arguments: urls, css, xpath"
|
||||
)
|
||||
if not urls:
|
||||
if css:
|
||||
urls = self.css(css)
|
||||
|
|
|
|||
|
|
@ -121,9 +121,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
return self.__class__(self)
|
||||
|
||||
def deepcopy(self):
|
||||
"""Return a `deep copy`_ of this item.
|
||||
|
||||
.. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
|
||||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst
|
|||
"""
|
||||
|
||||
|
||||
class Link(object):
|
||||
class Link:
|
||||
"""Link objects represent an extracted link by the LinkExtractor."""
|
||||
|
||||
__slots__ = ['url', 'text', 'fragment', 'nofollow']
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs)
|
|||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
|
||||
|
||||
|
||||
class FilteringLinkExtractor(object):
|
||||
class FilteringLinkExtractor:
|
||||
|
||||
_csstranslator = HTMLTranslator()
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ def _nons(tag):
|
|||
return tag
|
||||
|
||||
|
||||
class LxmlParserLinkExtractor(object):
|
||||
class LxmlParserLinkExtractor:
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True, canonicalized=False):
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def unbound_method(method):
|
|||
return method
|
||||
|
||||
|
||||
class ItemLoader(object):
|
||||
class ItemLoader:
|
||||
|
||||
default_item_class = Item
|
||||
default_input_processor = Identity()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter
|
|||
from scrapy.loader.common import wrap_loader_context
|
||||
|
||||
|
||||
class MapCompose(object):
|
||||
class MapCompose:
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
|
|
@ -36,7 +36,7 @@ class MapCompose(object):
|
|||
return values
|
||||
|
||||
|
||||
class Compose(object):
|
||||
class Compose:
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
|
|
@ -61,7 +61,7 @@ class Compose(object):
|
|||
return value
|
||||
|
||||
|
||||
class TakeFirst(object):
|
||||
class TakeFirst:
|
||||
|
||||
def __call__(self, values):
|
||||
for value in values:
|
||||
|
|
@ -69,13 +69,13 @@ class TakeFirst(object):
|
|||
return value
|
||||
|
||||
|
||||
class Identity(object):
|
||||
class Identity:
|
||||
|
||||
def __call__(self, values):
|
||||
return values
|
||||
|
||||
|
||||
class SelectJmes(object):
|
||||
class SelectJmes:
|
||||
"""
|
||||
Query the input string for the jmespath (given at instantiation),
|
||||
and return the answer
|
||||
|
|
@ -95,7 +95,7 @@ class SelectJmes(object):
|
|||
return self.compiled_path.search(value)
|
||||
|
||||
|
||||
class Join(object):
|
||||
class Join:
|
||||
|
||||
def __init__(self, separator=u' '):
|
||||
self.separator = separator
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
|
|||
DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
|
||||
|
||||
|
||||
class LogFormatter(object):
|
||||
class LogFormatter:
|
||||
"""Class for generating log messages for different actions.
|
||||
|
||||
All methods must return a dictionary listing the parameters ``level``, ``msg``
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from email.mime.text import MIMEText
|
|||
from email.utils import COMMASPACE, formatdate
|
||||
from io import BytesIO
|
||||
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
from twisted.internet import defer, ssl
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
|
@ -27,8 +27,7 @@ def _to_bytes_or_none(text):
|
|||
return to_bytes(text)
|
||||
|
||||
|
||||
class MailSender(object):
|
||||
|
||||
class MailSender:
|
||||
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
|
||||
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
|
||||
self.smtphost = smtphost
|
||||
|
|
@ -47,6 +46,7 @@ class MailSender(object):
|
|||
settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
|
||||
|
||||
def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
|
||||
from twisted.internet import reactor
|
||||
if attachs:
|
||||
msg = MIMEMultipart()
|
||||
else:
|
||||
|
|
@ -111,11 +111,12 @@ class MailSender(object):
|
|||
|
||||
def _sendmail(self, to_addrs, msg):
|
||||
# Import twisted.mail here because it is not available in python3
|
||||
from twisted.internet import reactor
|
||||
from twisted.mail.smtp import ESMTPSenderFactory
|
||||
msg = BytesIO(msg)
|
||||
d = defer.Deferred()
|
||||
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
|
||||
to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
|
||||
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
|
||||
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
|
||||
requireTransportSecurity=self.smtptls)
|
||||
factory.noisy = False
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MiddlewareManager(object):
|
||||
class MiddlewareManager:
|
||||
"""Base class for implementing middleware managers"""
|
||||
|
||||
component_name = 'foo middleware'
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class FileException(Exception):
|
|||
"""General media error exception"""
|
||||
|
||||
|
||||
class FSFilesStore(object):
|
||||
class FSFilesStore:
|
||||
def __init__(self, basedir):
|
||||
if '://' in basedir:
|
||||
basedir = basedir.split('://', 1)[1]
|
||||
|
|
@ -75,7 +75,7 @@ class FSFilesStore(object):
|
|||
seen.add(dirname)
|
||||
|
||||
|
||||
class S3FilesStore(object):
|
||||
class S3FilesStore:
|
||||
AWS_ACCESS_KEY_ID = None
|
||||
AWS_SECRET_ACCESS_KEY = None
|
||||
AWS_ENDPOINT_URL = None
|
||||
|
|
@ -213,7 +213,7 @@ class S3FilesStore(object):
|
|||
return extra
|
||||
|
||||
|
||||
class GCSFilesStore(object):
|
||||
class GCSFilesStore:
|
||||
|
||||
GCS_PROJECT_ID = None
|
||||
|
||||
|
|
@ -259,7 +259,7 @@ class GCSFilesStore(object):
|
|||
)
|
||||
|
||||
|
||||
class FTPFilesStore(object):
|
||||
class FTPFilesStore:
|
||||
|
||||
FTP_USERNAME = None
|
||||
FTP_PASSWORD = None
|
||||
|
|
@ -500,7 +500,7 @@ class FilesPipeline(MediaPipeline):
|
|||
spider.crawler.stats.inc_value('file_count', spider=spider)
|
||||
spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
|
||||
|
||||
### Overridable Interface
|
||||
# Overridable Interface
|
||||
def get_media_requests(self, item, info):
|
||||
return [Request(x) for x in item.get(self.files_urls_field, [])]
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from scrapy.utils.python import to_bytes
|
|||
from scrapy.http import Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.exceptions import DropItem
|
||||
#TODO: from scrapy.pipelines.media import MediaPipeline
|
||||
# TODO: from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.files import FileException, FilesPipeline
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import functools
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MediaPipeline(object):
|
||||
class MediaPipeline:
|
||||
|
||||
LOG_FAILED_RESULTS = True
|
||||
|
||||
class SpiderInfo(object):
|
||||
class SpiderInfo:
|
||||
def __init__(self, spider):
|
||||
self.spider = spider
|
||||
self.downloading = set()
|
||||
|
|
@ -141,24 +141,26 @@ class MediaPipeline(object):
|
|||
# This code fixes a memory leak by avoiding to keep references to
|
||||
# the Request and Response objects on the Media Pipeline cache.
|
||||
#
|
||||
# Twisted inline callbacks pass return values using the function
|
||||
# twisted.internet.defer.returnValue, which encapsulates the return
|
||||
# value inside a _DefGen_Return base exception.
|
||||
#
|
||||
# What happens when the media_downloaded callback raises another
|
||||
# What happens when the media_downloaded callback raises an
|
||||
# exception, for example a FileException('download-error') when
|
||||
# the Response status code is not 200 OK, is that it stores the
|
||||
# _DefGen_Return exception on the FileException context.
|
||||
# the Response status code is not 200 OK, is that the original
|
||||
# StopIteration exception (which in turn contains the failed
|
||||
# Response and by extension, the original Request) gets encapsulated
|
||||
# within the FileException context.
|
||||
#
|
||||
# Originally, Scrapy was using twisted.internet.defer.returnValue
|
||||
# inside functions decorated with twisted.internet.defer.inlineCallbacks,
|
||||
# encapsulating the returned Response in a _DefGen_Return exception
|
||||
# instead of a StopIteration.
|
||||
#
|
||||
# To avoid keeping references to the Response and therefore Request
|
||||
# objects on the Media Pipeline cache, we should wipe the context of
|
||||
# the exception encapsulated by the Twisted Failure when its a
|
||||
# _DefGen_Return instance.
|
||||
# the encapsulated exception when it is a StopIteration instance
|
||||
#
|
||||
# This problem does not occur in Python 2.7 since we don't have
|
||||
# Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
|
||||
context = getattr(result.value, '__context__', None)
|
||||
if isinstance(context, _DefGen_Return):
|
||||
if isinstance(context, StopIteration):
|
||||
setattr(result.value, '__context__', None)
|
||||
|
||||
info.downloading.remove(fp)
|
||||
|
|
@ -166,7 +168,7 @@ class MediaPipeline(object):
|
|||
for wad in info.waiting.pop(fp):
|
||||
defer_result(result).chainDeferred(wad)
|
||||
|
||||
### Overridable Interface
|
||||
# Overridable Interface
|
||||
def media_to_download(self, request, info):
|
||||
"""Check request before starting download"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ class ScrapyPriorityQueue:
|
|||
return sum(len(x) for x in self.queues.values()) if self.queues else 0
|
||||
|
||||
|
||||
class DownloaderInterface(object):
|
||||
class DownloaderInterface:
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.downloader = crawler.engine.downloader
|
||||
|
|
@ -129,8 +129,8 @@ class DownloaderInterface(object):
|
|||
return len(self.downloader.slots[slot].active)
|
||||
|
||||
|
||||
class DownloaderAwarePriorityQueue(object):
|
||||
""" PriorityQueue which takes Downloader activity in account:
|
||||
class DownloaderAwarePriorityQueue:
|
||||
""" PriorityQueue which takes Downloader activity into account:
|
||||
domains (slots) with the least amount of active downloads are dequeued
|
||||
first.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object
|
|||
from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
|
||||
|
||||
|
||||
class ResponseTypes(object):
|
||||
class ResponseTypes:
|
||||
|
||||
CLASSES = {
|
||||
'text/html': 'scrapy.http.HtmlResponse',
|
||||
|
|
@ -71,7 +71,7 @@ class ResponseTypes(object):
|
|||
cls = Response
|
||||
if b'Content-Type' in headers:
|
||||
cls = self.from_content_type(
|
||||
content_type=headers[b'Content-type'],
|
||||
content_type=headers[b'Content-Type'],
|
||||
content_encoding=headers.get(b'Content-Encoding')
|
||||
)
|
||||
if cls is Response and b'Content-Disposition' in headers:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def get_settings_priority(priority):
|
|||
return priority
|
||||
|
||||
|
||||
class SettingsAttribute(object):
|
||||
class SettingsAttribute:
|
||||
|
||||
"""Class for storing data related to settings attributes.
|
||||
|
||||
|
|
|
|||
|
|
@ -133,9 +133,8 @@ EXTENSIONS_BASE = {
|
|||
}
|
||||
|
||||
FEED_TEMPDIR = None
|
||||
FEED_URI = None
|
||||
FEEDS = {}
|
||||
FEED_URI_PARAMS = None # a function to extend uri arguments
|
||||
FEED_FORMAT = 'jsonlines'
|
||||
FEED_STORE_EMPTY = False
|
||||
FEED_EXPORT_ENCODING = None
|
||||
FEED_EXPORT_FIELDS = None
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
DEPRECATED_SETTINGS = [
|
||||
('TRACK_REFS', 'no longer needed (trackref is always enabled)'),
|
||||
('RESPONSE_CLASSES', 'no longer supported'),
|
||||
('DEFAULT_RESPONSE_ENCODING', 'no longer supported'),
|
||||
('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'),
|
||||
('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'),
|
||||
('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'),
|
||||
('SQLITE_DB', 'no longer supported'),
|
||||
('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
|
||||
('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
|
||||
]
|
||||
|
||||
|
||||
def check_deprecated_settings(settings):
|
||||
deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None]
|
||||
if deprecated:
|
||||
msg = "You are using the following settings which are deprecated or obsolete"
|
||||
msg += " (ask scrapy-users@googlegroups.com for alternatives):"
|
||||
msg = msg + "\n " + "\n ".join("%s: %s" % x for x in deprecated)
|
||||
warnings.warn(msg, ScrapyDeprecationWarning)
|
||||
|
|
@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config
|
|||
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
|
||||
|
||||
|
||||
class Shell(object):
|
||||
class Shell:
|
||||
|
||||
relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
|
||||
Settings)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from pydispatch import dispatcher
|
|||
from scrapy.utils import signal as _signal
|
||||
|
||||
|
||||
class SignalManager(object):
|
||||
class SignalManager:
|
||||
|
||||
def __init__(self, sender=dispatcher.Anonymous):
|
||||
self.sender = sender
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes
|
|||
|
||||
|
||||
@implementer(ISpiderLoader)
|
||||
class SpiderLoader(object):
|
||||
class SpiderLoader:
|
||||
"""
|
||||
SpiderLoader is a class which locates and loads spiders
|
||||
in a Scrapy project.
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy.http import Request
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DepthMiddleware(object):
|
||||
class DepthMiddleware:
|
||||
|
||||
def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
|
||||
self.maxdepth = maxdepth
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class HttpError(IgnoreRequest):
|
|||
super(HttpError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class HttpErrorMiddleware(object):
|
||||
class HttpErrorMiddleware:
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OffsiteMiddleware(object):
|
||||
class OffsiteMiddleware:
|
||||
|
||||
def __init__(self, stats):
|
||||
self.stats = stats
|
||||
|
|
@ -53,13 +53,22 @@ class OffsiteMiddleware(object):
|
|||
allowed_domains = getattr(spider, 'allowed_domains', None)
|
||||
if not allowed_domains:
|
||||
return re.compile('') # allow all by default
|
||||
url_pattern = re.compile("^https?://.*$")
|
||||
url_pattern = re.compile(r"^https?://.*$")
|
||||
port_pattern = re.compile(r":\d+$")
|
||||
domains = []
|
||||
for domain in allowed_domains:
|
||||
if url_pattern.match(domain):
|
||||
if domain is None:
|
||||
continue
|
||||
elif url_pattern.match(domain):
|
||||
message = ("allowed_domains accepts only domains, not URLs. "
|
||||
"Ignoring URL entry %s in allowed_domains." % domain)
|
||||
warnings.warn(message, URLWarning)
|
||||
domains = [re.escape(d) for d in allowed_domains if d is not None]
|
||||
elif port_pattern.search(domain):
|
||||
message = ("allowed_domains accepts only domains without ports. "
|
||||
"Ignoring entry %s in allowed_domains." % domain)
|
||||
warnings.warn(message, PortWarning)
|
||||
else:
|
||||
domains.append(re.escape(domain))
|
||||
regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
|
||||
return re.compile(regex)
|
||||
|
||||
|
|
@ -70,3 +79,7 @@ class OffsiteMiddleware(object):
|
|||
|
||||
class URLWarning(Warning):
|
||||
pass
|
||||
|
||||
|
||||
class PortWarning(Warning):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url"
|
|||
POLICY_SCRAPY_DEFAULT = "scrapy-default"
|
||||
|
||||
|
||||
class ReferrerPolicy(object):
|
||||
class ReferrerPolicy:
|
||||
|
||||
NOREFERRER_SCHEMES = LOCAL_SCHEMES
|
||||
|
||||
|
|
@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False):
|
|||
return None
|
||||
|
||||
|
||||
class RefererMiddleware(object):
|
||||
class RefererMiddleware:
|
||||
|
||||
def __init__(self, settings=None):
|
||||
self.default_policy = DefaultReferrerPolicy
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue