mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into allow-customizing-export-column-names
This commit is contained in:
Adrián Chaves
GitHub
commit
f9c63384c0
|
|
@ -1,4 +1,5 @@
|
|||
version: 2
|
||||
formats: all
|
||||
sphinx:
|
||||
configuration: docs/conf.py
|
||||
fail_on_warning: true
|
||||
|
|
|
|||
20
.travis.yml
20
.travis.yml
|
|
@ -11,23 +11,33 @@ matrix:
|
|||
python: 3.8
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- python: 3.5
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5.2
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5.2
|
||||
- python: 3.6
|
||||
- python: 3.7
|
||||
- env: PYPI_RELEASE_JOB=true
|
||||
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=extra-deps
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ including a list of features.
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 3.5+
|
||||
* Python 3.5.2+
|
||||
* Works on Linux, Windows, macOS, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
|
|
@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
|
|||
need to install watchdog (``pip install watchdog``) and then use::
|
||||
|
||||
make watch
|
||||
|
||||
Alternative method using tox
|
||||
----------------------------
|
||||
|
||||
To compile the documentation to HTML run the following command::
|
||||
|
||||
tox -e docs
|
||||
|
||||
Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Scrapy documentation build configuration file, created by
|
||||
# sphinx-quickstart on Mon Nov 24 12:02:52 2008.
|
||||
#
|
||||
|
|
@ -102,6 +100,9 @@ exclude_trees = ['.build']
|
|||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
|
||||
# List of Sphinx warnings that will not be raised
|
||||
suppress_warnings = ['epub.unknown_project_files']
|
||||
|
||||
|
||||
# Options for HTML output
|
||||
# -----------------------
|
||||
|
|
@ -280,6 +281,7 @@ coverage_ignore_pyobjects = [
|
|||
# -------------------------------------
|
||||
|
||||
intersphinx_mapping = {
|
||||
'attrs': ('https://www.attrs.org/en/stable/', None),
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
|
|
@ -302,3 +304,4 @@ hoverxref_role_types = {
|
|||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
|
|
|||
23
docs/faq.rst
23
docs/faq.rst
|
|
@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 3.5+
|
||||
Scrapy is supported under Python 3.5.2+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
|
|
@ -342,15 +342,15 @@ method for this purpose. For example::
|
|||
|
||||
from copy import deepcopy
|
||||
|
||||
from scrapy.item import BaseItem
|
||||
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
class MultiplyItemsMiddleware:
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
for item in result:
|
||||
if isinstance(item, (BaseItem, dict)):
|
||||
for _ in range(item['multiply_by']):
|
||||
if is_item(item):
|
||||
adapter = ItemAdapter(item)
|
||||
for _ in range(adapter['multiply_by']):
|
||||
yield deepcopy(item)
|
||||
|
||||
Does Scrapy support IPv6 addresses?
|
||||
|
|
@ -371,6 +371,19 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch
|
|||
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
|
||||
|
||||
|
||||
.. _faq-stop-response-download:
|
||||
|
||||
How can I cancel the download of a given response?
|
||||
--------------------------------------------------
|
||||
|
||||
In some situations, it might be useful to stop the download of a certain response.
|
||||
For instance, if you only need the first part of a large response and you would like
|
||||
to save resources by avoiding the download of the whole body.
|
||||
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
|
||||
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
|
||||
the :ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
|
||||
|
||||
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
|
||||
.. _user agents: https://en.wikipedia.org/wiki/User_agent
|
||||
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Installation guide
|
|||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 3.5 or above under CPython (default Python
|
||||
Scrapy runs on Python 3.5.2 or above under CPython (default Python
|
||||
implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares
|
|||
provided while constructing the crawler, and it is created after the
|
||||
arguments given in the :meth:`crawl` method.
|
||||
|
||||
.. method:: crawl(\*args, \**kwargs)
|
||||
.. method:: crawl(*args, **kwargs)
|
||||
|
||||
Starts the crawler by instantiating its spider class with the given
|
||||
``args`` and ``kwargs`` arguments, while setting the execution engine in
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ Spiders
|
|||
-------
|
||||
|
||||
Spiders are custom classes written by Scrapy users to parse responses and
|
||||
extract items (aka scraped items) from them or additional requests to
|
||||
extract :ref:`items <topics-items>` from them or additional requests to
|
||||
follow. For more information see :ref:`topics-spiders`.
|
||||
|
||||
.. _component-pipelines:
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ override three methods:
|
|||
|
||||
.. module:: scrapy.contracts
|
||||
|
||||
.. class:: Contract(method, \*args)
|
||||
.. class:: Contract(method, *args)
|
||||
|
||||
:param method: callback function to which the contract is associated
|
||||
:type method: function
|
||||
|
|
|
|||
|
|
@ -53,21 +53,28 @@ There are several use cases for coroutines in Scrapy. Code that would
|
|||
return Deferreds when written for previous Scrapy versions, such as downloader
|
||||
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
class DbPipeline:
|
||||
def _update_item(self, data, item):
|
||||
item['field'] = data
|
||||
adapter = ItemAdapter(item)
|
||||
adapter['field'] = data
|
||||
return item
|
||||
|
||||
def process_item(self, item, spider):
|
||||
dfd = db.get_some_data(item['id'])
|
||||
adapter = ItemAdapter(item)
|
||||
dfd = db.get_some_data(adapter['id'])
|
||||
dfd.addCallback(self._update_item, item)
|
||||
return dfd
|
||||
|
||||
becomes::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
class DbPipeline:
|
||||
async def process_item(self, item, spider):
|
||||
item['field'] = await db.get_some_data(item['id'])
|
||||
adapter = ItemAdapter(item)
|
||||
adapter['field'] = await db.get_some_data(adapter['id'])
|
||||
return item
|
||||
|
||||
Coroutines may be used to call asynchronous code. This includes other
|
||||
|
|
|
|||
|
|
@ -202,6 +202,11 @@ CookiesMiddleware
|
|||
sends them back on subsequent requests (from that spider), just like web
|
||||
browsers do.
|
||||
|
||||
.. caution:: When non-UTF8 encoded byte sequences are passed to a
|
||||
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
|
||||
a warning. Refer to :ref:`topics-logging-advanced-customization`
|
||||
to customize the logging behaviour.
|
||||
|
||||
The following settings can be used to configure the cookie middleware:
|
||||
|
||||
* :setting:`COOKIES_ENABLED`
|
||||
|
|
|
|||
|
|
@ -184,6 +184,18 @@ data from it:
|
|||
>>> json.loads(json_data)
|
||||
{'field': 'value'}
|
||||
|
||||
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
|
||||
|
||||
For example, if the JavaScript code contains
|
||||
``var data = {field: "value", secondField: "second value"};``
|
||||
you can extract that data as follows:
|
||||
|
||||
>>> import chompjs
|
||||
>>> javascript = response.css('script::text').get()
|
||||
>>> data = chompjs.parse_js_object(javascript)
|
||||
>>> data
|
||||
{'field': 'value', 'secondField': 'second value'}
|
||||
|
||||
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
|
||||
that you can parse using :ref:`selectors <topics-selectors>`.
|
||||
|
||||
|
|
@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration.
|
|||
|
||||
|
||||
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
|
||||
.. _curl: https://curl.haxx.se/
|
||||
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
|
||||
|
|
|
|||
|
|
@ -14,13 +14,6 @@ Built-in Exceptions reference
|
|||
|
||||
Here's a list of all exceptions included in Scrapy and their usage.
|
||||
|
||||
DropItem
|
||||
--------
|
||||
|
||||
.. exception:: DropItem
|
||||
|
||||
The exception that must be raised by item pipeline stages to stop processing an
|
||||
Item. For more information see :ref:`topics-item-pipeline`.
|
||||
|
||||
CloseSpider
|
||||
-----------
|
||||
|
|
@ -47,6 +40,14 @@ DontCloseSpider
|
|||
This exception can be raised in a :signal:`spider_idle` signal handler to
|
||||
prevent the spider from being closed.
|
||||
|
||||
DropItem
|
||||
--------
|
||||
|
||||
.. exception:: DropItem
|
||||
|
||||
The exception that must be raised by item pipeline stages to stop processing an
|
||||
Item. For more information see :ref:`topics-item-pipeline`.
|
||||
|
||||
IgnoreRequest
|
||||
-------------
|
||||
|
||||
|
|
@ -77,3 +78,37 @@ NotSupported
|
|||
|
||||
This exception is raised to indicate an unsupported feature.
|
||||
|
||||
StopDownload
|
||||
-------------
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
.. exception:: StopDownload(fail=True)
|
||||
|
||||
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
|
||||
indicate that no further bytes should be downloaded for a response.
|
||||
|
||||
The ``fail`` boolean parameter controls which method will handle the resulting
|
||||
response:
|
||||
|
||||
* If ``fail=True`` (default), the request errback is called. The response object is
|
||||
available as the ``response`` attribute of the ``StopDownload`` exception,
|
||||
which is in turn stored as the ``value`` attribute of the received
|
||||
:class:`~twisted.python.failure.Failure` object. This means that in an errback
|
||||
defined as ``def errback(self, failure)``, the response can be accessed though
|
||||
``failure.value.response``.
|
||||
|
||||
* If ``fail=False``, the request callback is called instead.
|
||||
|
||||
In both cases, the response could have its body truncated: the body contains
|
||||
all bytes received up until the exception is raised, including the bytes
|
||||
received in the signal handler that raises the exception. Also, the response
|
||||
object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
|
||||
attribute.
|
||||
|
||||
.. note:: ``fail`` is a keyword-only parameter, i.e. raising
|
||||
``StopDownload(False)`` or ``StopDownload(True)`` will raise
|
||||
a :class:`TypeError`.
|
||||
|
||||
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
|
||||
and the :ref:`topics-stop-response-download` topic for additional information and examples.
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
|
|||
Item Exporters to group scraped items to different files according to the
|
||||
value of one of their fields::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
|
||||
class PerYearXmlExportPipeline:
|
||||
|
|
@ -53,7 +54,8 @@ value of one of their fields::
|
|||
exporter.finish_exporting()
|
||||
|
||||
def _exporter_for_item(self, item):
|
||||
year = item['year']
|
||||
adapter = ItemAdapter(item)
|
||||
year = adapter['year']
|
||||
if year not in self.year_to_exporter:
|
||||
f = open('{}.xml'.format(year), 'wb')
|
||||
exporter = XmlItemExporter(f)
|
||||
|
|
@ -167,9 +169,10 @@ BaseItemExporter
|
|||
value unchanged except for ``unicode`` values which are encoded to
|
||||
``str`` using the encoding declared in the :attr:`encoding` attribute.
|
||||
|
||||
:param field: the field being serialized. If a raw dict is being
|
||||
exported (not :class:`~.Item`) *field* value is an empty dict.
|
||||
:type field: :class:`~scrapy.item.Field` object or an empty dict
|
||||
:param field: the field being serialized. If the source :ref:`item object
|
||||
<item-types>` does not define field metadata, *field* is an empty
|
||||
:class:`dict`.
|
||||
:type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
|
||||
|
||||
:param name: the name of the field being serialized
|
||||
:type name: str
|
||||
|
|
@ -207,10 +210,10 @@ BaseItemExporter
|
|||
{'field1': 'Field 1', 'field2': 'Field 2'}
|
||||
|
||||
.. [1] Not all exporters respect the specified field order.
|
||||
.. [2] If you yield items as dicts (not :class:`Item` instances),
|
||||
exporters that need to know the fields to export beforehand, like
|
||||
:class:`CsvItemExporter`, only export the fields found in the
|
||||
first item.
|
||||
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
||||
all their possible fields, exporters that do not support exporting
|
||||
a different subset of fields per item will only export the fields
|
||||
found in the first item exported.
|
||||
.. [3] Dicts preserve insertion order since `Python 3.7`_
|
||||
(`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
|
||||
of Python, use an OrderedDict_ to enforce a specific field order.
|
||||
|
|
@ -255,9 +258,9 @@ PythonItemExporter
|
|||
XmlItemExporter
|
||||
---------------
|
||||
|
||||
.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
|
||||
.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
|
||||
|
||||
Exports Items in XML format to the specified file object.
|
||||
Exports items in XML format to the specified file object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -309,9 +312,9 @@ XmlItemExporter
|
|||
CsvItemExporter
|
||||
---------------
|
||||
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
|
||||
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
|
||||
|
||||
Exports Items in CSV format to the given file-like object. If the
|
||||
Exports items in CSV format to the given file-like object. If the
|
||||
:attr:`fields_to_export` attribute is set, it will be used to define the
|
||||
CSV columns, their order and their column names. The
|
||||
:attr:`export_empty_fields` attribute has no effect on this exporter.
|
||||
|
|
@ -342,9 +345,9 @@ CsvItemExporter
|
|||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
|
||||
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
|
||||
|
||||
Exports Items in pickle format to the given file-like object.
|
||||
Exports items in pickle format to the given file-like object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -362,9 +365,9 @@ PickleItemExporter
|
|||
PprintItemExporter
|
||||
------------------
|
||||
|
||||
.. class:: PprintItemExporter(file, \**kwargs)
|
||||
.. class:: PprintItemExporter(file, **kwargs)
|
||||
|
||||
Exports Items in pretty print format to the specified file object.
|
||||
Exports items in pretty print format to the specified file object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -382,9 +385,9 @@ PprintItemExporter
|
|||
JsonItemExporter
|
||||
----------------
|
||||
|
||||
.. class:: JsonItemExporter(file, \**kwargs)
|
||||
.. class:: JsonItemExporter(file, **kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
Exports items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
|
|
@ -411,9 +414,9 @@ JsonItemExporter
|
|||
JsonLinesItemExporter
|
||||
---------------------
|
||||
|
||||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||
.. class:: JsonLinesItemExporter(file, **kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
Exports items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
|
|
|
|||
|
|
@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following
|
|||
|
||||
.. method:: process_item(self, item, spider)
|
||||
|
||||
This method is called for every item pipeline component. :meth:`process_item`
|
||||
must either: return a dict with data, return an :class:`~scrapy.item.Item`
|
||||
(or any descendant class) object, return a
|
||||
:class:`~twisted.internet.defer.Deferred` or raise
|
||||
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
|
||||
processed by further pipeline components.
|
||||
This method is called for every item pipeline component.
|
||||
|
||||
:param item: the item scraped
|
||||
:type item: :class:`~scrapy.item.Item` object or a dict
|
||||
`item` is an :ref:`item object <item-types>`, see
|
||||
:ref:`supporting-item-types`.
|
||||
|
||||
:meth:`process_item` must either: return an :ref:`item object <item-types>`,
|
||||
return a :class:`~twisted.internet.defer.Deferred` or raise a
|
||||
:exc:`~scrapy.exceptions.DropItem` exception.
|
||||
|
||||
Dropped items are no longer processed by further pipeline components.
|
||||
|
||||
:param item: the scraped item
|
||||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the
|
|||
(``price_excludes_vat`` attribute), and drops those items which don't
|
||||
contain a price::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class PricePipeline:
|
||||
|
||||
vat_factor = 1.15
|
||||
|
||||
def process_item(self, item, spider):
|
||||
if item.get('price'):
|
||||
if item.get('price_excludes_vat'):
|
||||
item['price'] = item['price'] * self.vat_factor
|
||||
adapter = ItemAdapter(item)
|
||||
if adapter.get('price'):
|
||||
if adapter.get('price_excludes_vat'):
|
||||
adapter['price'] = adapter['price'] * self.vat_factor
|
||||
return item
|
||||
else:
|
||||
raise DropItem("Missing price in %s" % item)
|
||||
|
|
@ -103,6 +108,8 @@ format::
|
|||
|
||||
import json
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
class JsonWriterPipeline:
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -112,7 +119,7 @@ format::
|
|||
self.file.close()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
line = json.dumps(dict(item)) + "\n"
|
||||
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
|
||||
self.file.write(line)
|
||||
return item
|
||||
|
||||
|
|
@ -131,6 +138,7 @@ The main point of this example is to show how to use :meth:`from_crawler`
|
|||
method and how to clean up the resources properly.::
|
||||
|
||||
import pymongo
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
class MongoPipeline:
|
||||
|
||||
|
|
@ -155,7 +163,7 @@ method and how to clean up the resources properly.::
|
|||
self.client.close()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
self.db[self.collection_name].insert_one(dict(item))
|
||||
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
|
||||
return item
|
||||
|
||||
.. _MongoDB: https://www.mongodb.com/
|
||||
|
|
@ -167,18 +175,21 @@ method and how to clean up the resources properly.::
|
|||
Take screenshot of item
|
||||
-----------------------
|
||||
|
||||
This example demonstrates how to return a
|
||||
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
|
||||
It uses Splash_ to render screenshot of item url. Pipeline
|
||||
makes request to locally running instance of Splash_. After request is downloaded,
|
||||
it saves the screenshot to a file and adds filename to the item.
|
||||
This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
|
||||
the :meth:`process_item` method.
|
||||
|
||||
This item pipeline makes a request to a locally-running instance of Splash_ to
|
||||
render a screenshot of the item URL. After the request response is downloaded,
|
||||
the item pipeline saves the screenshot to a file and adds the filename to the
|
||||
item.
|
||||
|
||||
::
|
||||
|
||||
import scrapy
|
||||
import hashlib
|
||||
from urllib.parse import quote
|
||||
|
||||
import scrapy
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
class ScreenshotPipeline:
|
||||
"""Pipeline that uses Splash to render screenshot of
|
||||
|
|
@ -187,7 +198,8 @@ it saves the screenshot to a file and adds filename to the item.
|
|||
SPLASH_URL = "http://localhost:8050/render.png?url={}"
|
||||
|
||||
async def process_item(self, item, spider):
|
||||
encoded_item_url = quote(item["url"])
|
||||
adapter = ItemAdapter(item)
|
||||
encoded_item_url = quote(adapter["url"])
|
||||
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
|
||||
request = scrapy.Request(screenshot_url)
|
||||
response = await spider.crawler.engine.download(request, spider)
|
||||
|
|
@ -197,14 +209,14 @@ it saves the screenshot to a file and adds filename to the item.
|
|||
return item
|
||||
|
||||
# Save screenshot to file, filename will be hash of url.
|
||||
url = item["url"]
|
||||
url = adapter["url"]
|
||||
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
|
||||
filename = "{}.png".format(url_hash)
|
||||
with open(filename, "wb") as f:
|
||||
f.write(response.body)
|
||||
|
||||
# Store filename in item.
|
||||
item["screenshot_filename"] = filename
|
||||
adapter["screenshot_filename"] = filename
|
||||
return item
|
||||
|
||||
.. _Splash: https://splash.readthedocs.io/en/stable/
|
||||
|
|
@ -217,6 +229,7 @@ already processed. Let's say that our items have a unique id, but our spider
|
|||
returns multiples items with the same id::
|
||||
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
class DuplicatesPipeline:
|
||||
|
|
@ -225,10 +238,11 @@ returns multiples items with the same id::
|
|||
self.ids_seen = set()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
if item['id'] in self.ids_seen:
|
||||
raise DropItem("Duplicate item found: %s" % item)
|
||||
adapter = ItemAdapter(item)
|
||||
if adapter['id'] in self.ids_seen:
|
||||
raise DropItem("Duplicate item found: %r" % item)
|
||||
else:
|
||||
self.ids_seen.add(item['id'])
|
||||
self.ids_seen.add(adapter['id'])
|
||||
return item
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,29 +8,155 @@ Items
|
|||
:synopsis: Item and Field classes
|
||||
|
||||
The main goal in scraping is to extract structured data from unstructured
|
||||
sources, typically, web pages. Scrapy spiders can return the extracted data
|
||||
as Python dicts. While convenient and familiar, Python dicts lack structure:
|
||||
it is easy to make a typo in a field name or return inconsistent data,
|
||||
especially in a larger project with many spiders.
|
||||
sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
|
||||
extracted data as `items`, Python objects that define key-value pairs.
|
||||
|
||||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide an API similar to :class:`dict` API with a convenient syntax
|
||||
for declaring their available fields.
|
||||
Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
|
||||
item, you may use whichever type of item you want. When you write code that
|
||||
receives an item, your code should :ref:`work for any item type
|
||||
<supporting-item-types>`.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
serialization can be customized using Item fields metadata, :mod:`trackref`
|
||||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
.. _item-types:
|
||||
|
||||
Item Types
|
||||
==========
|
||||
|
||||
Scrapy supports the following types of items, via the `itemadapter`_ library:
|
||||
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
|
||||
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
|
||||
|
||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||
|
||||
.. _dict-items:
|
||||
|
||||
Dictionaries
|
||||
------------
|
||||
|
||||
As an item type, :class:`dict` is convenient and familiar.
|
||||
|
||||
.. _item-objects:
|
||||
|
||||
Item objects
|
||||
------------
|
||||
|
||||
:class:`Item` provides a :class:`dict`-like API plus additional features that
|
||||
make it the most feature-complete item type:
|
||||
|
||||
.. class:: Item([arg])
|
||||
|
||||
:class:`Item` objects replicate the standard :class:`dict` API, including
|
||||
its ``__init__`` method.
|
||||
|
||||
:class:`Item` allows defining field names, so that:
|
||||
|
||||
- :class:`KeyError` is raised when using undefined field names (i.e.
|
||||
prevents typos going unnoticed)
|
||||
|
||||
- :ref:`Item exporters <topics-exporters>` can export all fields by
|
||||
default even if the first scraped object does not have values for all
|
||||
of them
|
||||
|
||||
:class:`Item` also allows defining field metadata, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`).
|
||||
|
||||
:class:`Item` objects also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
||||
.. automethod:: deepcopy
|
||||
|
||||
.. attribute:: fields
|
||||
|
||||
A dictionary containing *all declared fields* for this Item, not only
|
||||
those populated. The keys are the field names and the values are the
|
||||
:class:`Field` objects used in the :ref:`Item declaration
|
||||
<topics-items-declaring>`.
|
||||
|
||||
Example::
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
class CustomItem(Item):
|
||||
one_field = Field()
|
||||
another_field = Field()
|
||||
|
||||
.. _dataclass-items:
|
||||
|
||||
Dataclass objects
|
||||
-----------------
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
:func:`~dataclasses.dataclass` allows defining item classes with field names,
|
||||
so that :ref:`item exporters <topics-exporters>` can export all fields by
|
||||
default even if the first scraped object does not have values for all of them.
|
||||
|
||||
Additionally, ``dataclass`` items also allow to:
|
||||
|
||||
* define the type and default value of each defined field.
|
||||
|
||||
* define custom field metadata through :func:`dataclasses.field`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
They work natively in Python 3.7 or later, or using the `dataclasses
|
||||
backport`_ in Python 3.6.
|
||||
|
||||
.. _dataclasses backport: https://pypi.org/project/dataclasses/
|
||||
|
||||
Example::
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class CustomItem:
|
||||
one_field: str
|
||||
another_field: int
|
||||
|
||||
.. note:: Field types are not enforced at run time.
|
||||
|
||||
.. _attrs-items:
|
||||
|
||||
attr.s objects
|
||||
--------------
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
:func:`attr.s` allows defining item classes with field names,
|
||||
so that :ref:`item exporters <topics-exporters>` can export all fields by
|
||||
default even if the first scraped object does not have values for all of them.
|
||||
|
||||
Additionally, ``attr.s`` items also allow to:
|
||||
|
||||
* define the type and default value of each defined field.
|
||||
|
||||
* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
|
||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||
|
||||
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
|
||||
|
||||
Example::
|
||||
|
||||
import attr
|
||||
|
||||
@attr.s
|
||||
class CustomItem:
|
||||
one_field = attr.ib()
|
||||
another_field = attr.ib()
|
||||
|
||||
|
||||
Working with Item objects
|
||||
=========================
|
||||
|
||||
.. _topics-items-declaring:
|
||||
|
||||
Declaring Items
|
||||
===============
|
||||
Declaring Item subclasses
|
||||
-------------------------
|
||||
|
||||
Items are declared using a simple class definition syntax and :class:`Field`
|
||||
objects. Here is an example::
|
||||
Item subclasses are declared using a simple class definition syntax and
|
||||
:class:`Field` objects. Here is an example::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
|
@ -48,10 +174,11 @@ objects. Here is an example::
|
|||
.. _Django: https://www.djangoproject.com/
|
||||
.. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
|
||||
|
||||
|
||||
.. _topics-items-fields:
|
||||
|
||||
Item Fields
|
||||
===========
|
||||
Declaring fields
|
||||
----------------
|
||||
|
||||
:class:`Field` objects are used to specify metadata for each field. For
|
||||
example, the serializer function for the ``last_updated`` field illustrated in
|
||||
|
|
@ -72,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item
|
|||
do not stay assigned as class attributes. Instead, they can be accessed through
|
||||
the :attr:`Item.fields` attribute.
|
||||
|
||||
Working with Items
|
||||
==================
|
||||
.. class:: Field([arg])
|
||||
|
||||
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
|
||||
doesn't provide any extra functionality or attributes. In other words,
|
||||
:class:`Field` objects are plain-old Python dicts. A separate class is used
|
||||
to support the :ref:`item declaration syntax <topics-items-declaring>`
|
||||
based on class attributes.
|
||||
|
||||
.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
|
||||
items. Please refer to the documentation for `dataclasses.field`_ and
|
||||
`attr.ib`_ for additional information.
|
||||
|
||||
.. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
|
||||
.. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
|
||||
|
||||
|
||||
Working with Item objects
|
||||
-------------------------
|
||||
|
||||
Here are some examples of common tasks performed with items, using the
|
||||
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
|
||||
notice the API is very similar to the :class:`dict` API.
|
||||
|
||||
Creating items
|
||||
--------------
|
||||
''''''''''''''
|
||||
|
||||
>>> product = Product(name='Desktop PC', price=1000)
|
||||
>>> print(product)
|
||||
|
|
@ -88,7 +231,7 @@ Product(name='Desktop PC', price=1000)
|
|||
|
||||
|
||||
Getting field values
|
||||
--------------------
|
||||
''''''''''''''''''''
|
||||
|
||||
>>> product['name']
|
||||
Desktop PC
|
||||
|
|
@ -128,7 +271,7 @@ False
|
|||
|
||||
|
||||
Setting field values
|
||||
--------------------
|
||||
''''''''''''''''''''
|
||||
|
||||
>>> product['last_updated'] = 'today'
|
||||
>>> product['last_updated']
|
||||
|
|
@ -141,7 +284,7 @@ KeyError: 'Product does not support field: lala'
|
|||
|
||||
|
||||
Accessing all populated values
|
||||
------------------------------
|
||||
''''''''''''''''''''''''''''''
|
||||
|
||||
To access all populated values, just use the typical :class:`dict` API:
|
||||
|
||||
|
|
@ -155,7 +298,7 @@ To access all populated values, just use the typical :class:`dict` API:
|
|||
.. _copying-items:
|
||||
|
||||
Copying items
|
||||
-------------
|
||||
'''''''''''''
|
||||
|
||||
To copy an item, you must first decide whether you want a shallow copy or a
|
||||
deep copy.
|
||||
|
|
@ -183,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
|
|||
|
||||
|
||||
Other common tasks
|
||||
------------------
|
||||
''''''''''''''''''
|
||||
|
||||
Creating dicts from items:
|
||||
|
||||
|
|
@ -201,8 +344,8 @@ Traceback (most recent call last):
|
|||
KeyError: 'Product does not support field: lala'
|
||||
|
||||
|
||||
Extending Items
|
||||
===============
|
||||
Extending Item subclasses
|
||||
-------------------------
|
||||
|
||||
You can extend Items (to add more fields or to change some metadata for some
|
||||
fields) by declaring a subclass of your original Item.
|
||||
|
|
@ -222,41 +365,25 @@ appending more values, or changing existing values, like this::
|
|||
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
|
||||
keeping all the previously existing metadata values.
|
||||
|
||||
Item objects
|
||||
============
|
||||
|
||||
.. class:: Item([arg])
|
||||
.. _supporting-item-types:
|
||||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
Supporting All Item Types
|
||||
=========================
|
||||
|
||||
Items replicate the standard :class:`dict` API, including its ``__init__``
|
||||
method, and also provide the following additional API members:
|
||||
In code that receives an item, such as methods of :ref:`item pipelines
|
||||
<topics-item-pipeline>` or :ref:`spider middlewares
|
||||
<topics-spider-middleware>`, it is a good practice to use the
|
||||
:class:`~itemadapter.ItemAdapter` class and the
|
||||
:func:`~itemadapter.is_item` function to write code that works for
|
||||
any :ref:`supported item type <item-types>`:
|
||||
|
||||
.. automethod:: copy
|
||||
.. autoclass:: itemadapter.ItemAdapter
|
||||
|
||||
.. automethod:: deepcopy
|
||||
.. autofunction:: itemadapter.is_item
|
||||
|
||||
.. attribute:: fields
|
||||
|
||||
A dictionary containing *all declared fields* for this Item, not only
|
||||
those populated. The keys are the field names and the values are the
|
||||
:class:`Field` objects used in the :ref:`Item declaration
|
||||
<topics-items-declaring>`.
|
||||
|
||||
Field objects
|
||||
=============
|
||||
|
||||
.. class:: Field([arg])
|
||||
|
||||
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
|
||||
doesn't provide any extra functionality or attributes. In other words,
|
||||
:class:`Field` objects are plain-old Python dicts. A separate class is used
|
||||
to support the :ref:`item declaration syntax <topics-items-declaring>`
|
||||
based on class attributes.
|
||||
|
||||
Other classes related to Item
|
||||
=============================
|
||||
|
||||
.. autoclass:: BaseItem
|
||||
Other classes related to items
|
||||
==============================
|
||||
|
||||
.. autoclass:: ItemMeta
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
Debugging memory leaks
|
||||
======================
|
||||
|
||||
In Scrapy, objects such as Requests, Responses and Items have a finite
|
||||
In Scrapy, objects such as requests, responses and items have a finite
|
||||
lifetime: they are created, used for a while, and finally destroyed.
|
||||
|
||||
From all those objects, the Request is probably the one with the longest
|
||||
|
|
@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
|
|||
========================================
|
||||
|
||||
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
|
||||
memory leaks. It basically tracks the references to all live Requests,
|
||||
Responses, Item and Selector objects.
|
||||
memory leaks. It basically tracks the references to all live Request,
|
||||
Response, Item, Spider and Selector objects.
|
||||
|
||||
You can enter the telnet console and inspect how many objects (of the classes
|
||||
mentioned above) are currently alive using the ``prefs()`` function which is an
|
||||
|
|
@ -200,11 +200,10 @@ Debugging memory leaks with muppy
|
|||
|
||||
``trackref`` provides a very convenient mechanism for tracking down memory
|
||||
leaks, but it only keeps track of the objects that are more likely to cause
|
||||
memory leaks (Requests, Responses, Items, and Selectors). However, there are
|
||||
other cases where the memory leaks could come from other (more or less obscure)
|
||||
objects. If this is your case, and you can't find your leaks using ``trackref``,
|
||||
you still have another resource: the muppy library.
|
||||
|
||||
memory leaks. However, there are other cases where the memory leaks could come
|
||||
from other (more or less obscure) objects. If this is your case, and you can't
|
||||
find your leaks using ``trackref``, you still have another resource: the muppy
|
||||
library.
|
||||
|
||||
You can use muppy from `Pympler`_.
|
||||
|
||||
|
|
|
|||
|
|
@ -7,13 +7,12 @@ Item Loaders
|
|||
.. module:: scrapy.loader
|
||||
:synopsis: Item Loader class
|
||||
|
||||
Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
|
||||
<topics-items>`. Even though Items can be populated using their own
|
||||
dictionary-like API, Item Loaders provide a much more convenient API for
|
||||
populating them from a scraping process, by automating some common tasks like
|
||||
parsing the raw extracted data before assigning it.
|
||||
Item Loaders provide a convenient mechanism for populating scraped :ref:`items
|
||||
<topics-items>`. Even though items can be populated directly, Item Loaders provide a
|
||||
much more convenient API for populating them from a scraping process, by automating
|
||||
some common tasks like parsing the raw extracted data before assigning it.
|
||||
|
||||
In other words, :ref:`Items <topics-items>` provide the *container* of
|
||||
In other words, :ref:`items <topics-items>` provide the *container* of
|
||||
scraped data, while Item Loaders provide the mechanism for *populating* that
|
||||
container.
|
||||
|
||||
|
|
@ -25,10 +24,10 @@ Using Item Loaders to populate items
|
|||
====================================
|
||||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
|
||||
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
|
||||
using the Item class specified in the :attr:`ItemLoader.default_item_class`
|
||||
attribute.
|
||||
instantiate it with an :ref:`item object <topics-items>` or without one, in which
|
||||
case an instance of :class:`~scrapy.item.Item` is automatically created in the
|
||||
Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
|
||||
specified in the :attr:`ItemLoader.default_item_class` attribute.
|
||||
|
||||
Then, you start collecting values into the Item Loader, typically using
|
||||
:ref:`Selectors <topics-selectors>`. You can add more than one value to
|
||||
|
|
@ -88,7 +87,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
|
|||
:meth:`~ItemLoader.add_value` methods) and the result of the input processor is
|
||||
collected and kept inside the ItemLoader. After collecting all data, the
|
||||
:meth:`ItemLoader.load_item` method is called to populate and get the populated
|
||||
:class:`~scrapy.item.Item` object. That's when the output processor is
|
||||
:ref:`item object <topics-items>`. That's when the output processor is
|
||||
called with the data previously collected (and processed using the input
|
||||
processor). The result of the output processor is the final value that gets
|
||||
assigned to the item.
|
||||
|
|
@ -153,12 +152,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
|||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
||||
Item Loaders are declared like Items, by using a class definition syntax. Here
|
||||
is an example::
|
||||
Item Loaders are declared using a class definition syntax. Here is an example::
|
||||
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.processors import TakeFirst, MapCompose, Join
|
||||
|
|
@ -273,11 +270,11 @@ There are several ways to modify Item Loader context values:
|
|||
ItemLoader objects
|
||||
==================
|
||||
|
||||
.. class:: ItemLoader([item, selector, response], \**kwargs)
|
||||
.. class:: ItemLoader([item, selector, response], **kwargs)
|
||||
|
||||
Return a new Item Loader for populating the given Item. If no item is
|
||||
given, one is instantiated automatically using the class in
|
||||
:attr:`default_item_class`.
|
||||
Return a new Item Loader for populating the given :ref:`item object
|
||||
<topics-items>`. If no item object is given, one is instantiated
|
||||
automatically using the class in :attr:`default_item_class`.
|
||||
|
||||
When instantiated with a ``selector`` or a ``response`` parameters
|
||||
the :class:`ItemLoader` class provides convenient mechanisms for extracting
|
||||
|
|
@ -286,7 +283,7 @@ ItemLoader objects
|
|||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: :class:`~scrapy.item.Item` object
|
||||
:type item: :ref:`item object <topics-items>`
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
|
||||
|
|
@ -303,7 +300,7 @@ ItemLoader objects
|
|||
|
||||
:class:`ItemLoader` instances have the following methods:
|
||||
|
||||
.. method:: get_value(value, \*processors, \**kwargs)
|
||||
.. method:: get_value(value, *processors, **kwargs)
|
||||
|
||||
Process the given ``value`` by the given ``processors`` and keyword
|
||||
arguments.
|
||||
|
|
@ -321,7 +318,7 @@ ItemLoader objects
|
|||
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
|
||||
'FOO`
|
||||
|
||||
.. method:: add_value(field_name, value, \*processors, \**kwargs)
|
||||
.. method:: add_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Process and then add the given ``value`` for the given field.
|
||||
|
||||
|
|
@ -343,11 +340,11 @@ ItemLoader objects
|
|||
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
|
||||
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
|
||||
|
||||
.. method:: replace_value(field_name, value, \*processors, \**kwargs)
|
||||
.. method:: replace_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_value` but replaces the collected data with the
|
||||
new value instead of adding it.
|
||||
.. method:: get_xpath(xpath, \*processors, \**kwargs)
|
||||
.. method:: get_xpath(xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
|
|
@ -367,7 +364,7 @@ ItemLoader objects
|
|||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
|
||||
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
|
|
@ -385,12 +382,12 @@ ItemLoader objects
|
|||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
|
||||
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_xpath` but replaces collected data instead of
|
||||
adding it.
|
||||
|
||||
.. method:: get_css(css, \*processors, \**kwargs)
|
||||
.. method:: get_css(css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
|
|
@ -410,7 +407,7 @@ ItemLoader objects
|
|||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_css(field_name, css, \*processors, \**kwargs)
|
||||
.. method:: add_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
|
|
@ -428,7 +425,7 @@ ItemLoader objects
|
|||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_css('price', 'p#price', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_css(field_name, css, \*processors, \**kwargs)
|
||||
.. method:: replace_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_css` but replaces collected data instead of
|
||||
adding it.
|
||||
|
|
@ -444,17 +441,19 @@ ItemLoader objects
|
|||
|
||||
Create a nested loader with an xpath selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
|
||||
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
|
||||
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: nested_css(css)
|
||||
|
||||
Create a nested loader with a css selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
|
||||
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
|
||||
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: get_collected_values(field_name)
|
||||
|
||||
|
|
@ -477,7 +476,7 @@ ItemLoader objects
|
|||
|
||||
.. attribute:: item
|
||||
|
||||
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
|
||||
The :ref:`item object <topics-items>` being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
|
|
@ -488,8 +487,8 @@ ItemLoader objects
|
|||
|
||||
.. attribute:: default_item_class
|
||||
|
||||
An Item class (or factory), used to instantiate items when not given in
|
||||
the ``__init__`` method.
|
||||
An :ref:`item object <topics-items>` class or factory, used to
|
||||
instantiate items when not given in the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
|
|
@ -678,7 +677,7 @@ Here is a list of all built-in processors:
|
|||
>>> proc(['one', 'two', 'three'])
|
||||
'one<br>two<br>three'
|
||||
|
||||
.. class:: Compose(\*functions, \**default_loader_context)
|
||||
.. class:: Compose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions. This means that each input value of this processor is passed to
|
||||
|
|
@ -706,7 +705,7 @@ Here is a list of all built-in processors:
|
|||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
attribute.
|
||||
|
||||
.. class:: MapCompose(\*functions, \**default_loader_context)
|
||||
.. class:: MapCompose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions, similar to the :class:`Compose` processor. The difference with
|
||||
|
|
|
|||
|
|
@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending
|
|||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
||||
|
||||
.. _topics-logging-advanced-customization:
|
||||
|
||||
Advanced customization
|
||||
----------------------
|
||||
|
||||
|
|
@ -262,7 +265,6 @@ scrapy.utils.log module
|
|||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
||||
import logging
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
logging.basicConfig(
|
||||
filename='log.txt',
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ this:
|
|||
4. When the files are downloaded, another field (``files``) will be populated
|
||||
with the results. This field will contain a list of dicts with information
|
||||
about the downloaded files, such as the downloaded path, the original
|
||||
scraped url (taken from the ``file_urls`` field) , and the file checksum.
|
||||
scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
|
||||
The files in the list of the ``files`` field will retain the same order of
|
||||
the original ``file_urls`` field. If some file failed downloading, an
|
||||
error will be logged and the file won't be present in the ``files`` field.
|
||||
|
|
@ -156,7 +156,7 @@ following forms::
|
|||
|
||||
ftp://username:password@address:port/path
|
||||
ftp://address:port/path
|
||||
|
||||
|
||||
If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
|
||||
:setting:`FTP_PASSWORD` settings respectively.
|
||||
|
||||
|
|
@ -243,20 +243,22 @@ Usage example
|
|||
.. setting:: IMAGES_URLS_FIELD
|
||||
.. setting:: IMAGES_RESULT_FIELD
|
||||
|
||||
In order to use a media pipeline first, :ref:`enable it
|
||||
In order to use a media pipeline, first :ref:`enable it
|
||||
<topics-media-pipeline-enabling>`.
|
||||
|
||||
Then, if a spider returns a dict with the URLs key (``file_urls`` or
|
||||
``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
|
||||
put the results under respective key (``files`` or ``images``).
|
||||
Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
|
||||
field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
|
||||
respectively), the pipeline will put the results under the respective field
|
||||
(``files`` or ``images``).
|
||||
|
||||
If you prefer to use :class:`~.Item`, then define a custom item with the
|
||||
necessary fields, like in this example for Images Pipeline::
|
||||
When using :ref:`item types <item-types>` for which fields are defined beforehand,
|
||||
you must define both the URLs field and the results field. For example, when
|
||||
using the images pipeline, items must define both the ``image_urls`` and the
|
||||
``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
|
||||
|
||||
import scrapy
|
||||
|
||||
class MyItem(scrapy.Item):
|
||||
|
||||
# ... other item fields ...
|
||||
image_urls = scrapy.Field()
|
||||
images = scrapy.Field()
|
||||
|
|
@ -445,8 +447,11 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
:meth:`~get_media_requests` method and return a Request for each
|
||||
file URL::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
for file_url in item['file_urls']:
|
||||
adapter = ItemAdapter(item)
|
||||
for file_url in adapter['file_urls']:
|
||||
yield scrapy.Request(file_url)
|
||||
|
||||
Those requests will be processed by the pipeline and, when they have finished
|
||||
|
|
@ -470,6 +475,14 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
* ``checksum`` - a `MD5 hash`_ of the image contents
|
||||
|
||||
* ``status`` - the file status indication. It can be one of the following:
|
||||
|
||||
* ``downloaded`` - file was downloaded.
|
||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||
according to the file expiration policy.
|
||||
* ``cached`` - file was already scheduled for download, by another item
|
||||
sharing the same file.
|
||||
|
||||
The list of tuples received by :meth:`~item_completed` is
|
||||
guaranteed to retain the same order of the requests returned from the
|
||||
:meth:`~get_media_requests` method.
|
||||
|
|
@ -479,7 +492,8 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
[(True,
|
||||
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
|
||||
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
|
||||
'url': 'http://www.example.com/files/product1.pdf'}),
|
||||
'url': 'http://www.example.com/files/product1.pdf',
|
||||
'status': 'downloaded'}),
|
||||
(False,
|
||||
Failure(...))]
|
||||
|
||||
|
|
@ -500,13 +514,15 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
store the downloaded file paths (passed in results) in the ``file_paths``
|
||||
item field, and we drop the item if it doesn't contain any files::
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exceptions import DropItem
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
file_paths = [x['path'] for ok, x in results if ok]
|
||||
if not file_paths:
|
||||
raise DropItem("Item contains no files")
|
||||
item['file_paths'] = file_paths
|
||||
adapter = ItemAdapter(item)
|
||||
adapter['file_paths'] = file_paths
|
||||
return item
|
||||
|
||||
By default, the :meth:`item_completed` method returns the item.
|
||||
|
|
@ -580,8 +596,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
|
|||
above::
|
||||
|
||||
import scrapy
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from itemadapter import ItemAdapter
|
||||
from scrapy.exceptions import DropItem
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
|
||||
class MyImagesPipeline(ImagesPipeline):
|
||||
|
||||
|
|
@ -593,7 +610,8 @@ above::
|
|||
image_paths = [x['path'] for ok, x in results if ok]
|
||||
if not image_paths:
|
||||
raise DropItem("Item contains no images")
|
||||
item['image_paths'] = image_paths
|
||||
adapter = ItemAdapter(item)
|
||||
adapter['image_paths'] = image_paths
|
||||
return item
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -189,6 +189,10 @@ Request objects
|
|||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
|
||||
|
||||
In case of a failure to process the request, this dict can be accessed as
|
||||
``failure.request.cb_kwargs`` in the request's errback. For more information,
|
||||
see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
Return a new Request which is a copy of this Request. See also:
|
||||
|
|
@ -312,6 +316,31 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
----------------------------------------------
|
||||
|
||||
In case of a failure to process the request, you may be interested in
|
||||
accessing arguments to the callback functions so you can process further
|
||||
based on the arguments in the errback. The following example shows how to
|
||||
achieve this by using ``Failure.request.cb_kwargs``::
|
||||
|
||||
def parse(self, response):
|
||||
request = scrapy.Request('http://www.example.com/index.html',
|
||||
callback=self.parse_page2,
|
||||
errback=self.errback_page2,
|
||||
cb_kwargs=dict(main_url=response.url))
|
||||
yield request
|
||||
|
||||
def parse_page2(self, response, main_url):
|
||||
pass
|
||||
|
||||
def errback_page2(self, failure):
|
||||
yield dict(
|
||||
main_url=failure.request.cb_kwargs['main_url'],
|
||||
)
|
||||
|
||||
.. _topics-request-meta:
|
||||
|
||||
Request.meta special keys
|
||||
|
|
@ -385,6 +414,51 @@ The meta key is used set retry times per request. When initialized, the
|
|||
:reqmeta:`max_retry_times` meta key takes higher precedence over the
|
||||
:setting:`RETRY_TIMES` setting.
|
||||
|
||||
|
||||
.. _topics-stop-response-download:
|
||||
|
||||
Stopping the download of a Response
|
||||
===================================
|
||||
|
||||
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
|
||||
:class:`~scrapy.signals.bytes_received` signal handler will stop the
|
||||
download of a given response. See the following example::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class StopSpider(scrapy.Spider):
|
||||
name = "stop"
|
||||
start_urls = ["https://docs.scrapy.org/en/latest/"]
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
spider = super().from_crawler(crawler)
|
||||
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
|
||||
return spider
|
||||
|
||||
def parse(self, response):
|
||||
# 'last_chars' show that the full response was not downloaded
|
||||
yield {"len": len(response.text), "last_chars": response.text[-40:]}
|
||||
|
||||
def on_bytes_received(self, data, request, spider):
|
||||
raise scrapy.exceptions.StopDownload(fail=False)
|
||||
|
||||
which produces the following output::
|
||||
|
||||
2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
|
||||
2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
|
||||
2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
|
||||
2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
|
||||
2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
|
||||
{'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n \n <title>Scr'}
|
||||
2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
|
||||
|
||||
By default, resulting responses are handled by their corresponding errbacks. To
|
||||
call their callback instead, like in this example, pass ``fail=False`` to the
|
||||
:exc:`~scrapy.exceptions.StopDownload` exception.
|
||||
|
||||
|
||||
.. _topics-request-response-ref-request-subclasses:
|
||||
|
||||
Request subclasses
|
||||
|
|
@ -716,9 +790,9 @@ Response objects
|
|||
.. versionadded:: 2.1.0
|
||||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. method:: Response.copy()
|
||||
|
|
@ -834,10 +908,10 @@ TextResponse objects
|
|||
|
||||
.. automethod:: TextResponse.follow_all
|
||||
|
||||
.. method:: TextResponse.body_as_unicode()
|
||||
.. automethod:: TextResponse.json()
|
||||
|
||||
The same as :attr:`text`, but available as a method. This method is
|
||||
kept for backward compatibility; please prefer ``response.text``.
|
||||
Returns a Python object from deserialized JSON document.
|
||||
The result is cached after the first call.
|
||||
|
||||
|
||||
HtmlResponse objects
|
||||
|
|
|
|||
|
|
@ -236,8 +236,8 @@ CONCURRENT_ITEMS
|
|||
|
||||
Default: ``100``
|
||||
|
||||
Maximum number of concurrent items (per response) to process in parallel in the
|
||||
Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
|
||||
Maximum number of concurrent items (per response) to process in parallel in
|
||||
:ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS
|
||||
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ engine_started
|
|||
|
||||
Sent when the Scrapy engine has started crawling.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
.. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
|
||||
depending on how the spider was started. So **don't** rely on this signal
|
||||
|
|
@ -127,7 +127,7 @@ engine_stopped
|
|||
Sent when the Scrapy engine is stopped (for example, when a crawling
|
||||
process has finished).
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
Item signals
|
||||
------------
|
||||
|
|
@ -149,10 +149,10 @@ item_scraped
|
|||
Sent when an item has been scraped, after it has passed all the
|
||||
:ref:`topics-item-pipeline` stages (without being dropped).
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param item: the item scraped
|
||||
:type item: dict or :class:`~scrapy.item.Item` object
|
||||
:param item: the scraped item
|
||||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -169,10 +169,10 @@ item_dropped
|
|||
Sent after an item has been dropped from the :ref:`topics-item-pipeline`
|
||||
when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param item: the item dropped from the :ref:`topics-item-pipeline`
|
||||
:type item: dict or :class:`~scrapy.item.Item` object
|
||||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param spider: the spider which scraped the item
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -194,10 +194,10 @@ item_error
|
|||
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
|
||||
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param item: the item dropped from the :ref:`topics-item-pipeline`
|
||||
:type item: dict or :class:`~scrapy.item.Item` object
|
||||
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
|
||||
:type item: :ref:`item object <item-types>`
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -220,7 +220,7 @@ spider_closed
|
|||
Sent after a spider has been closed. This can be used to release per-spider
|
||||
resources reserved on :signal:`spider_opened`.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has been closed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -244,7 +244,7 @@ spider_opened
|
|||
reserve per-spider resources, but can be used for any task that needs to be
|
||||
performed when a spider is opened.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
This signal supports returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has been opened
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -268,7 +268,7 @@ spider_idle
|
|||
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
|
||||
prevent the spider from being closed.
|
||||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param spider: the spider which has gone idle
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -287,7 +287,7 @@ spider_error
|
|||
|
||||
Sent when a spider callback generates an error (i.e. raises an exception).
|
||||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param failure: the exception raised
|
||||
:type failure: twisted.python.failure.Failure
|
||||
|
|
@ -310,7 +310,7 @@ request_scheduled
|
|||
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
|
||||
downloaded later.
|
||||
|
||||
The signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached the scheduler
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
|
@ -327,7 +327,7 @@ request_dropped
|
|||
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
|
||||
downloaded later, is rejected by the scheduler.
|
||||
|
||||
The signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached the scheduler
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
|
@ -343,7 +343,7 @@ request_reached_downloader
|
|||
|
||||
Sent when a :class:`~scrapy.http.Request` reached downloader.
|
||||
|
||||
The signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param request: the request that reached downloader
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
|
@ -370,6 +370,36 @@ request_left_downloader
|
|||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
bytes_received
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
.. signal:: bytes_received
|
||||
.. function:: bytes_received(data, request, spider)
|
||||
|
||||
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
|
||||
received for a specific request. This signal might be fired multiple
|
||||
times for the same request, with partial data each time. For instance,
|
||||
a possible scenario for a 25 kb response would be two signals fired
|
||||
with 10 kb of data, and a final one with 5 kb of data.
|
||||
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param data: the data received by the download handler
|
||||
:type data: :class:`bytes` object
|
||||
|
||||
:param request: the request that generated the download
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
||||
:param spider: the spider associated with the response
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. note:: Handlers of this signal can stop the download of a response while it
|
||||
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
|
||||
exception. Please refer to the :ref:`topics-stop-response-download` topic
|
||||
for additional information and examples.
|
||||
|
||||
Response signals
|
||||
----------------
|
||||
|
||||
|
|
@ -382,7 +412,7 @@ response_received
|
|||
Sent when the engine receives a new :class:`~scrapy.http.Response` from the
|
||||
downloader.
|
||||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param response: the response received
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
@ -401,7 +431,7 @@ response_downloaded
|
|||
|
||||
Sent by the downloader right after a ``HTTPResponse`` is downloaded.
|
||||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
This signal does not support returning deferreds from its handlers.
|
||||
|
||||
:param response: the response downloaded
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
|
|
|||
|
|
@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
it has processed the response.
|
||||
|
||||
:meth:`process_spider_output` must return an iterable of
|
||||
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
|
||||
objects.
|
||||
:class:`~scrapy.http.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
|
||||
:param response: the response which generated this output from the
|
||||
spider
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param result: the result returned by the spider
|
||||
:type result: an iterable of :class:`~scrapy.http.Request`, dict
|
||||
or :class:`~scrapy.item.Item` objects
|
||||
:type result: an iterable of :class:`~scrapy.http.Request` objects and
|
||||
:ref:`item object <topics-items>`
|
||||
|
||||
:param spider: the spider whose result is being processed
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
|
||||
.. method:: process_spider_exception(response, exception, spider)
|
||||
|
||||
This method is called when a spider or :meth:`process_spider_output`
|
||||
method (from a previous spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.http.Request`, dict or
|
||||
:class:`~scrapy.item.Item` objects.
|
||||
iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
|
||||
<topics-items>`.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
executing any other :meth:`process_spider_exception` in the following
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this:
|
|||
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
|
||||
Requests.
|
||||
|
||||
2. In the callback function, you parse the response (web page) and return either
|
||||
dicts with extracted data, :class:`~scrapy.item.Item` objects,
|
||||
2. In the callback function, you parse the response (web page) and return
|
||||
:ref:`item objects <topics-items>`,
|
||||
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
|
||||
Those Requests will also contain a callback (maybe
|
||||
the same) and will then be downloaded by Scrapy and then their
|
||||
|
|
@ -121,7 +121,7 @@ scrapy.Spider
|
|||
send log messages through it as described on
|
||||
:ref:`topics-logging-from-spiders`.
|
||||
|
||||
.. method:: from_crawler(crawler, \*args, \**kwargs)
|
||||
.. method:: from_crawler(crawler, *args, **kwargs)
|
||||
|
||||
This is the class method used by Scrapy to create your spiders.
|
||||
|
||||
|
|
@ -179,8 +179,8 @@ scrapy.Spider
|
|||
the same requirements as the :class:`Spider` class.
|
||||
|
||||
This method, as well as any other Request callback, must return an
|
||||
iterable of :class:`~scrapy.http.Request` and/or
|
||||
dicts or :class:`~scrapy.item.Item` objects.
|
||||
iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
|
||||
<topics-items>`.
|
||||
|
||||
:param response: the response to parse
|
||||
:type response: :class:`~scrapy.http.Response`
|
||||
|
|
@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback::
|
|||
yield scrapy.Request(response.urljoin(href), self.parse)
|
||||
|
||||
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
|
||||
to give data more structure you can use :ref:`topics-items`::
|
||||
to give data more structure you can use :class:`~scrapy.item.Item` objects::
|
||||
|
||||
import scrapy
|
||||
from myproject.items import MyItem
|
||||
|
|
@ -364,7 +364,7 @@ CrawlSpider
|
|||
|
||||
This method is called for the start_urls responses. It allows to parse
|
||||
the initial responses and must return either an
|
||||
:class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
|
||||
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
|
||||
object, or an iterable containing any of them.
|
||||
|
||||
Crawling rules
|
||||
|
|
@ -383,7 +383,7 @@ Crawling rules
|
|||
object with that name will be used) to be called for each link extracted with
|
||||
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
|
||||
as its first argument and must return either a single instance or an iterable of
|
||||
:class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
|
||||
:ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
|
||||
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
|
||||
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
|
||||
in its ``meta`` dictionary (under the ``link_text`` key)
|
||||
|
|
@ -531,7 +531,7 @@ XMLFeedSpider
|
|||
(``itertag``). Receives the response and an
|
||||
:class:`~scrapy.selector.Selector` for each node. Overriding this
|
||||
method is mandatory. Otherwise, you spider won't work. This method
|
||||
must return either a :class:`~scrapy.item.Item` object, a
|
||||
must return an :ref:`item object <topics-items>`, a
|
||||
:class:`~scrapy.http.Request` object, or an iterable containing any of
|
||||
them.
|
||||
|
||||
|
|
@ -541,7 +541,7 @@ XMLFeedSpider
|
|||
spider, and it's intended to perform any last time processing required
|
||||
before returning the results to the framework core, for example setting the
|
||||
item IDs. It receives a list of results and the response which originated
|
||||
those results. It must return a list of results (Items or Requests).
|
||||
those results. It must return a list of results (items or requests).
|
||||
|
||||
|
||||
XMLFeedSpider example
|
||||
|
|
|
|||
|
|
@ -14,50 +14,57 @@ Author: dufferzafar
|
|||
|
||||
import re
|
||||
|
||||
# Used for remembering the file (and its contents)
|
||||
# so we don't have to open the same file again.
|
||||
_filename = None
|
||||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
def main():
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
with open("build/linkcheck/output.txt") as out:
|
||||
output_lines = out.readlines()
|
||||
except IOError:
|
||||
print("linkcheck output not found; please run linkcheck first.")
|
||||
exit(1)
|
||||
# Used for remembering the file (and its contents)
|
||||
# so we don't have to open the same file again.
|
||||
_filename = None
|
||||
_contents = None
|
||||
|
||||
# For every line, fix the respective file
|
||||
for line in output_lines:
|
||||
match = re.match(line_re, line)
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
if match:
|
||||
newfilename = match.group(1)
|
||||
errortype = match.group(2)
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
with open("build/linkcheck/output.txt") as out:
|
||||
output_lines = out.readlines()
|
||||
except IOError:
|
||||
print("linkcheck output not found; please run linkcheck first.")
|
||||
exit(1)
|
||||
|
||||
# Broken links can't be fixed and
|
||||
# I am not sure what do with the local ones.
|
||||
if errortype.lower() in ["broken", "local"]:
|
||||
print("Not Fixed: " + line)
|
||||
# For every line, fix the respective file
|
||||
for line in output_lines:
|
||||
match = re.match(line_re, line)
|
||||
|
||||
if match:
|
||||
newfilename = match.group(1)
|
||||
errortype = match.group(2)
|
||||
|
||||
# Broken links can't be fixed and
|
||||
# I am not sure what do with the local ones.
|
||||
if errortype.lower() in ["broken", "local"]:
|
||||
print("Not Fixed: " + line)
|
||||
else:
|
||||
# If this is a new file
|
||||
if newfilename != _filename:
|
||||
|
||||
# Update the previous file
|
||||
if _filename:
|
||||
with open(_filename, "w") as _file:
|
||||
_file.write(_contents)
|
||||
|
||||
_filename = newfilename
|
||||
|
||||
# Read the new file to memory
|
||||
with open(_filename) as _file:
|
||||
_contents = _file.read()
|
||||
|
||||
_contents = _contents.replace(match.group(3), match.group(4))
|
||||
else:
|
||||
# If this is a new file
|
||||
if newfilename != _filename:
|
||||
# We don't understand what the current line means!
|
||||
print("Not Understood: " + line)
|
||||
|
||||
# Update the previous file
|
||||
if _filename:
|
||||
with open(_filename, "w") as _file:
|
||||
_file.write(_contents)
|
||||
|
||||
_filename = newfilename
|
||||
|
||||
# Read the new file to memory
|
||||
with open(_filename) as _file:
|
||||
_contents = _file.read()
|
||||
|
||||
_contents = _contents.replace(match.group(3), match.group(4))
|
||||
else:
|
||||
# We don't understand what the current line means!
|
||||
print("Not Understood: " + line)
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
[MASTER]
|
||||
persistent=no
|
||||
jobs=1 # >1 hides results
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable=abstract-method,
|
||||
anomalous-backslash-in-string,
|
||||
arguments-differ,
|
||||
attribute-defined-outside-init,
|
||||
bad-classmethod-argument,
|
||||
bad-continuation,
|
||||
bad-indentation,
|
||||
bad-mcs-classmethod-argument,
|
||||
bad-super-call,
|
||||
bad-whitespace,
|
||||
bare-except,
|
||||
blacklisted-name,
|
||||
broad-except,
|
||||
c-extension-no-member,
|
||||
catching-non-exception,
|
||||
cell-var-from-loop,
|
||||
comparison-with-callable,
|
||||
consider-iterating-dictionary,
|
||||
consider-using-in,
|
||||
consider-using-set-comprehension,
|
||||
consider-using-sys-exit,
|
||||
cyclic-import,
|
||||
dangerous-default-value,
|
||||
deprecated-method,
|
||||
deprecated-module,
|
||||
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
|
||||
eval-used,
|
||||
expression-not-assigned,
|
||||
fixme,
|
||||
function-redefined,
|
||||
global-statement,
|
||||
import-error,
|
||||
import-outside-toplevel,
|
||||
import-self,
|
||||
inconsistent-return-statements,
|
||||
inherit-non-class,
|
||||
invalid-name,
|
||||
invalid-overridden-method,
|
||||
isinstance-second-argument-not-valid-type,
|
||||
keyword-arg-before-vararg,
|
||||
line-too-long,
|
||||
logging-format-interpolation,
|
||||
logging-not-lazy,
|
||||
lost-exception,
|
||||
method-hidden,
|
||||
misplaced-comparison-constant,
|
||||
missing-docstring,
|
||||
missing-final-newline,
|
||||
multiple-imports,
|
||||
multiple-statements,
|
||||
no-else-continue,
|
||||
no-else-raise,
|
||||
no-else-return,
|
||||
no-init,
|
||||
no-member,
|
||||
no-method-argument,
|
||||
no-name-in-module,
|
||||
no-self-argument,
|
||||
no-self-use,
|
||||
no-value-for-parameter,
|
||||
not-an-iterable,
|
||||
not-callable,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
protected-access,
|
||||
redefined-argument-from-local,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
reimported,
|
||||
signature-differs,
|
||||
singleton-comparison,
|
||||
super-init-not-called,
|
||||
superfluous-parens,
|
||||
too-few-public-methods,
|
||||
too-many-ancestors,
|
||||
too-many-arguments,
|
||||
too-many-branches,
|
||||
too-many-format-args,
|
||||
too-many-function-args,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
trailing-newlines,
|
||||
trailing-whitespace,
|
||||
unbalanced-tuple-unpacking,
|
||||
undefined-variable,
|
||||
undefined-loop-variable,
|
||||
unexpected-special-method-signature,
|
||||
ungrouped-imports,
|
||||
unidiomatic-typecheck,
|
||||
unnecessary-comprehension,
|
||||
unnecessary-lambda,
|
||||
unnecessary-pass,
|
||||
unreachable,
|
||||
unsubscriptable-object,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-variable,
|
||||
unused-wildcard-import,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance, # Required for Python 2 support
|
||||
useless-return,
|
||||
useless-super-delegation,
|
||||
wildcard-import,
|
||||
wrong-import-order,
|
||||
wrong-import-position
|
||||
233
pytest.ini
233
pytest.ini
|
|
@ -20,232 +20,23 @@ addopts =
|
|||
twisted = 1
|
||||
markers =
|
||||
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
|
||||
flake8-max-line-length = 119
|
||||
flake8-ignore =
|
||||
W503
|
||||
# Files that are only meant to provide top-level imports are expected not
|
||||
# to use any of their imports:
|
||||
|
||||
# Exclude files that are meant to provide top-level imports
|
||||
# E402: Module level import not at top of file
|
||||
# F401: Module imported but unused
|
||||
scrapy/__init__.py E402
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/linkextractors/__init__.py E402 F401
|
||||
scrapy/selector/__init__.py F401
|
||||
scrapy/spiders/__init__.py E402 F401
|
||||
|
||||
# Issues pending a review:
|
||||
# extras
|
||||
extras/qps-bench-server.py E501
|
||||
extras/qpsclient.py E501 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E501 E128
|
||||
scrapy/commands/genspider.py E128 E501
|
||||
scrapy/commands/parse.py E128 E501
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501
|
||||
scrapy/commands/startproject.py E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E128
|
||||
scrapy/core/spidermw.py E501 E126
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501
|
||||
scrapy/core/downloader/tls.py E501
|
||||
scrapy/core/downloader/webclient.py E501 E128 E126
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E501 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E128
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128
|
||||
scrapy/http/response/text.py E501 E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E501 E402
|
||||
scrapy/linkextractors/lxmlhtml.py E501
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E501
|
||||
scrapy/pipelines/files.py E116 E501
|
||||
scrapy/pipelines/images.py E501
|
||||
scrapy/pipelines/media.py E501
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403
|
||||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E114 E116
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py E501 E129
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/asyncio.py E501
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/conf.py E402 E501
|
||||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E501
|
||||
scrapy/utils/gz.py E501
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501
|
||||
scrapy/utils/log.py E128 E501
|
||||
scrapy/utils/markup.py F403
|
||||
scrapy/utils/misc.py E501
|
||||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E501
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/test.py E501
|
||||
scrapy/utils/url.py E501 F403 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/cmdline.py E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E501
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501
|
||||
scrapy/mail.py E402 E128 E501
|
||||
scrapy/middleware.py E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E501
|
||||
scrapy/responsetypes.py E128 E501
|
||||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E501 E126
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501
|
||||
tests/test_closespider.py E501
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128
|
||||
tests/test_crawl.py E501 E741
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E128 E501 E126
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126
|
||||
tests/test_downloadermiddleware_decompression.py E501
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E501 E128
|
||||
tests/test_exporters.py E501 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E128 E128 E126
|
||||
tests/test_http_response.py E501 E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E741 E128 E117
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
tests/test_pipeline_files.py E501
|
||||
tests/test_pipeline_images.py F841 E501
|
||||
tests/test_pipeline_media.py E501 E741 E128
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126
|
||||
tests/test_selector.py E501
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E124 E501 E121
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501
|
||||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128
|
||||
tests/test_utils_iterators.py E501 E128 E129
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_url.py E501 E501 E126
|
||||
tests/test_webclient.py E501 E128 E122 E402 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
tests/test_utils_misc/__init__.py E501
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
|
|
|
|||
|
|
@ -2,33 +2,11 @@
|
|||
Scrapy - a web crawling and web scraping framework written for Python
|
||||
"""
|
||||
|
||||
__all__ = ['__version__', 'version_info', 'twisted_version',
|
||||
'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
|
||||
|
||||
# Scrapy version
|
||||
import pkgutil
|
||||
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
|
||||
version_info = tuple(int(v) if v.isdigit() else v
|
||||
for v in __version__.split('.'))
|
||||
del pkgutil
|
||||
|
||||
# Check minimum required Python version
|
||||
import sys
|
||||
if sys.version_info < (3, 5):
|
||||
print("Scrapy %s requires Python 3.5" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore noisy twisted deprecation warnings
|
||||
import warnings
|
||||
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
||||
del warnings
|
||||
|
||||
# Apply monkey patches to fix issues in external libraries
|
||||
from scrapy import _monkeypatches
|
||||
del _monkeypatches
|
||||
|
||||
from twisted import version as _txv
|
||||
twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
||||
|
||||
# Declare top-level shortcuts
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
|
|||
from scrapy.selector import Selector
|
||||
from scrapy.item import Item, Field
|
||||
|
||||
|
||||
__all__ = [
|
||||
'__version__', 'version_info', 'twisted_version', 'Spider',
|
||||
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
|
||||
]
|
||||
|
||||
|
||||
# Scrapy and Twisted versions
|
||||
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
|
||||
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
|
||||
twisted_version = (_txv.major, _txv.minor, _txv.micro)
|
||||
|
||||
|
||||
# Check minimum required Python version
|
||||
if sys.version_info < (3, 5, 2):
|
||||
print("Scrapy %s requires Python 3.5.2" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# Ignore noisy twisted deprecation warnings
|
||||
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
||||
|
||||
|
||||
del pkgutil
|
||||
del sys
|
||||
del warnings
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
import copyreg
|
||||
|
||||
|
||||
# Undo what Twisted's perspective broker adds to pickle register
|
||||
# to prevent bugs like Twisted#7989 while serializing requests
|
||||
import twisted.persisted.styles # NOQA
|
||||
# Remove only entries with twisted serializers for non-twisted types.
|
||||
for k, v in frozenset(copyreg.dispatch_table.items()):
|
||||
if not str(getattr(k, '__module__', '')).startswith('twisted') \
|
||||
and str(getattr(v, '__module__', '')).startswith('twisted'):
|
||||
copyreg.dispatch_table.pop(k)
|
||||
|
|
@ -165,6 +165,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
execute()
|
||||
finally:
|
||||
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
|
||||
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
|
||||
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
|
||||
# http://doc.pypy.org/en/latest/cpython_differences.html
|
||||
# ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
|
||||
garbage_collect()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import os
|
|||
from optparse import OptionGroup
|
||||
from twisted.python import failure
|
||||
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
|
|
@ -59,17 +59,17 @@ class ScrapyCommand:
|
|||
"""
|
||||
group = OptionGroup(parser, "Global Options")
|
||||
group.add_option("--logfile", metavar="FILE",
|
||||
help="log file. if omitted stderr will be used")
|
||||
help="log file. if omitted stderr will be used")
|
||||
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
|
||||
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
|
||||
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
|
||||
group.add_option("--nolog", action="store_true",
|
||||
help="disable logging completely")
|
||||
help="disable logging completely")
|
||||
group.add_option("--profile", metavar="FILE", default=None,
|
||||
help="write python cProfile stats to FILE")
|
||||
help="write python cProfile stats to FILE")
|
||||
group.add_option("--pidfile", metavar="FILE",
|
||||
help="write process ID to FILE")
|
||||
help="write process ID to FILE")
|
||||
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set/override setting (may be repeated)")
|
||||
help="set/override setting (may be repeated)")
|
||||
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
parser.add_option_group(group)
|
||||
|
|
@ -104,3 +104,27 @@ class ScrapyCommand:
|
|||
Entry point for running commands
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class BaseRunSpiderCommand(ScrapyCommand):
|
||||
"""
|
||||
Common class used to share functionality between the crawl and runspider commands
|
||||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.commands import BaseRunSpiderCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
class Command(BaseRunSpiderCommand):
|
||||
|
||||
requires_project = True
|
||||
|
||||
|
|
@ -13,25 +12,6 @@ class Command(ScrapyCommand):
|
|||
def short_desc(self):
|
||||
return "Run a spider"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ class Command(ScrapyCommand):
|
|||
parser.add_option("--spider", dest="spider", help="use this spider")
|
||||
parser.add_option("--headers", dest="headers", action="store_true",
|
||||
help="print response HTTP headers instead of body")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def _print_headers(self, headers, prefix):
|
||||
for key, values in headers.items():
|
||||
|
|
|
|||
|
|
@ -36,15 +36,15 @@ class Command(ScrapyCommand):
|
|||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List available templates")
|
||||
help="List available templates")
|
||||
parser.add_option("-e", "--edit", dest="edit", action="store_true",
|
||||
help="Edit spider after creating it")
|
||||
help="Edit spider after creating it")
|
||||
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
|
||||
help="Dump template to standard output")
|
||||
help="Dump template to standard output")
|
||||
parser.add_option("-t", "--template", dest="template", default="basic",
|
||||
help="Uses a custom template.")
|
||||
help="Uses a custom template.")
|
||||
parser.add_option("--force", dest="force", action="store_true",
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
help="If the spider already exists, overwrite it with the template")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.list:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import json
|
||||
import logging
|
||||
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.utils import display
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
|
||||
|
|
@ -33,29 +33,29 @@ class Command(ScrapyCommand):
|
|||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
help="don't show links to follow (extracted requests)")
|
||||
help="don't show links to follow (extracted requests)")
|
||||
parser.add_option("--noitems", dest="noitems", action="store_true",
|
||||
help="don't show scraped items")
|
||||
help="don't show scraped items")
|
||||
parser.add_option("--nocolour", dest="nocolour", action="store_true",
|
||||
help="avoid using pygments to colorize the output")
|
||||
help="avoid using pygments to colorize the output")
|
||||
parser.add_option("-r", "--rules", dest="rules", action="store_true",
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("--cbkwargs", dest="cbkwargs",
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
help="print each depth level one by one")
|
||||
help="print each depth level one by one")
|
||||
|
||||
@property
|
||||
def max_level(self):
|
||||
|
|
@ -81,7 +81,7 @@ class Command(ScrapyCommand):
|
|||
items = self.items.get(lvl, [])
|
||||
|
||||
print("# Scraped Items ", "-" * 60)
|
||||
display.pprint([dict(x) for x in items], colorize=colour)
|
||||
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
|
||||
|
||||
def print_requests(self, lvl=None, colour=True):
|
||||
if lvl is None:
|
||||
|
|
@ -117,7 +117,7 @@ class Command(ScrapyCommand):
|
|||
items, requests = [], []
|
||||
|
||||
for x in iterate_spider_output(callback(response, **cb_kwargs)):
|
||||
if isinstance(x, (BaseItem, dict)):
|
||||
if is_item(x):
|
||||
items.append(x)
|
||||
elif isinstance(x, Request):
|
||||
requests.append(x)
|
||||
|
|
|
|||
|
|
@ -3,9 +3,8 @@ import os
|
|||
from importlib import import_module
|
||||
|
||||
from scrapy.utils.spider import iter_spider_classes
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.commands import BaseRunSpiderCommand
|
||||
|
||||
|
||||
def _import_file(filepath):
|
||||
|
|
@ -24,7 +23,7 @@ def _import_file(filepath):
|
|||
return module
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
class Command(BaseRunSpiderCommand):
|
||||
|
||||
requires_project = False
|
||||
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
|
||||
|
|
@ -38,25 +37,6 @@ class Command(ScrapyCommand):
|
|||
def long_desc(self):
|
||||
return "Run the spider defined in the given file"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) != 1:
|
||||
raise UsageError()
|
||||
|
|
|
|||
|
|
@ -19,15 +19,15 @@ class Command(ScrapyCommand):
|
|||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--get", dest="get", metavar="SETTING",
|
||||
help="print raw setting value")
|
||||
help="print raw setting value")
|
||||
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
|
||||
help="print setting value, interpreted as a boolean")
|
||||
help="print setting value, interpreted as a boolean")
|
||||
parser.add_option("--getint", dest="getint", metavar="SETTING",
|
||||
help="print setting value, interpreted as an integer")
|
||||
help="print setting value, interpreted as an integer")
|
||||
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
|
||||
help="print setting value, interpreted as a float")
|
||||
help="print setting value, interpreted as a float")
|
||||
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
|
||||
help="print setting value, interpreted as a list")
|
||||
help="print setting value, interpreted as a list")
|
||||
|
||||
def run(self, args, opts):
|
||||
settings = self.crawler_process.settings
|
||||
|
|
|
|||
|
|
@ -34,11 +34,11 @@ class Command(ScrapyCommand):
|
|||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-c", dest="code",
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
help="evaluate the code in the shell, print the result and exit")
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
|
||||
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
help="use this spider")
|
||||
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
|
||||
help="do not handle HTTP 3xx status codes and print response as-is")
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import re
|
||||
import os
|
||||
import stat
|
||||
import string
|
||||
from importlib import import_module
|
||||
from os.path import join, exists, abspath
|
||||
|
|
@ -78,6 +79,29 @@ class Command(ScrapyCommand):
|
|||
else:
|
||||
copy2(srcname, dstname)
|
||||
copystat(src, dst)
|
||||
self._set_rw_permissions(dst)
|
||||
|
||||
def _set_rw_permissions(self, path):
|
||||
"""
|
||||
Sets permissions of a directory tree to +rw and +rwx for folders.
|
||||
This is necessary if the start template files come without write
|
||||
permissions.
|
||||
"""
|
||||
mode_rw = (stat.S_IRUSR
|
||||
| stat.S_IWUSR
|
||||
| stat.S_IRGRP
|
||||
| stat.S_IROTH)
|
||||
|
||||
mode_x = (stat.S_IXUSR
|
||||
| stat.S_IXGRP
|
||||
| stat.S_IXOTH)
|
||||
|
||||
os.chmod(path, mode_rw | mode_x)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for dir in dirs:
|
||||
os.chmod(join(root, dir), mode_rw | mode_x)
|
||||
for file in files:
|
||||
os.chmod(join(root, file), mode_rw)
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) not in (1, 2):
|
||||
|
|
@ -102,10 +126,8 @@ class Command(ScrapyCommand):
|
|||
move(join(project_dir, 'module'), join(project_dir, project_name))
|
||||
for paths in TEMPLATES_TO_RENDER:
|
||||
path = join(*paths)
|
||||
tplfile = join(project_dir,
|
||||
string.Template(path).substitute(project_name=project_name))
|
||||
render_templatefile(tplfile, project_name=project_name,
|
||||
ProjectName=string_camelcase(project_name))
|
||||
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
|
||||
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
|
||||
print("New Scrapy project '%s', using template directory '%s', "
|
||||
"created in:" % (project_name, self.templates_dir))
|
||||
print(" %s\n" % abspath(project_dir))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class Command(ScrapyCommand):
|
|||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
help="also display twisted/python/platform info (useful for bug reports)")
|
||||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import json
|
||||
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.http import Request
|
||||
from scrapy.exceptions import ContractFail
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.contracts import Contract
|
||||
from scrapy.exceptions import ContractFail
|
||||
from scrapy.http import Request
|
||||
|
||||
|
||||
# contracts
|
||||
|
|
@ -48,11 +48,11 @@ class ReturnsContract(Contract):
|
|||
"""
|
||||
|
||||
name = 'returns'
|
||||
objects = {
|
||||
'request': Request,
|
||||
'requests': Request,
|
||||
'item': (BaseItem, dict),
|
||||
'items': (BaseItem, dict),
|
||||
object_type_verifiers = {
|
||||
'request': lambda x: isinstance(x, Request),
|
||||
'requests': lambda x: isinstance(x, Request),
|
||||
'item': is_item,
|
||||
'items': is_item,
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
|
@ -64,7 +64,7 @@ class ReturnsContract(Contract):
|
|||
% len(self.args)
|
||||
)
|
||||
self.obj_name = self.args[0] or None
|
||||
self.obj_type = self.objects[self.obj_name]
|
||||
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
|
||||
|
||||
try:
|
||||
self.min_bound = int(self.args[1])
|
||||
|
|
@ -79,7 +79,7 @@ class ReturnsContract(Contract):
|
|||
def post_process(self, output):
|
||||
occurrences = 0
|
||||
for x in output:
|
||||
if isinstance(x, self.obj_type):
|
||||
if self.obj_type_verifier(x):
|
||||
occurrences += 1
|
||||
|
||||
assertion = (self.min_bound <= occurrences <= self.max_bound)
|
||||
|
|
@ -103,8 +103,8 @@ class ScrapesContract(Contract):
|
|||
|
||||
def post_process(self, output):
|
||||
for x in output:
|
||||
if isinstance(x, (BaseItem, dict)):
|
||||
missing = [arg for arg in self.args if arg not in x]
|
||||
if is_item(x):
|
||||
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
|
||||
if missing:
|
||||
raise ContractFail(
|
||||
"Missing fields: %s" % ", ".join(missing))
|
||||
missing_str = ", ".join(missing)
|
||||
raise ContractFail("Missing fields: %s" % missing_str)
|
||||
|
|
|
|||
|
|
@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
#
|
||||
# * getattr() for `_ssl_method` attribute for context factories
|
||||
# not calling super(..., self).__init__
|
||||
return CertificateOptions(verify=False,
|
||||
method=getattr(self, 'method',
|
||||
getattr(self, '_ssl_method', None)),
|
||||
fixBrokenPeers=True,
|
||||
acceptableCiphers=self.tls_ciphers)
|
||||
return CertificateOptions(
|
||||
verify=False,
|
||||
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
|
||||
fixBrokenPeers=True,
|
||||
acceptableCiphers=self.tls_ciphers,
|
||||
)
|
||||
|
||||
# kept for old-style HTTP/1.0 downloader context twisted calls,
|
||||
# e.g. connectSSL()
|
||||
|
|
@ -86,8 +87,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
|||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
# by default, since CAcert.org CA certificate is seldom shipped.
|
||||
return optionsForClientTLS(hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={
|
||||
'method': self._ssl_method,
|
||||
})
|
||||
return optionsForClientTLS(
|
||||
hostname=hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={'method': self._ssl_method},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -86,10 +86,9 @@ class FTPDownloadHandler:
|
|||
password = request.meta.get("ftp_password", self.default_password)
|
||||
passive_mode = 1 if bool(request.meta.get("ftp_passive",
|
||||
self.passive_mode)) else 0
|
||||
creator = ClientCreator(reactor, FTPClient, user, password,
|
||||
passive=passive_mode)
|
||||
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
|
||||
request, unquote(parsed_url.path))
|
||||
creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
|
||||
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
|
||||
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
|
||||
|
||||
def gotClient(self, client, request, filepath):
|
||||
self.client = client
|
||||
|
|
|
|||
|
|
@ -12,15 +12,17 @@ from urllib.parse import urldefrag
|
|||
from twisted.internet import defer, protocol, ssl
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
|
||||
from twisted.web.http import _DataLoss, PotentialDataLoss
|
||||
from twisted.web.http_headers import Headers as TxHeaders
|
||||
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
|
||||
from zope.interface import implementer
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.core.downloader.tls import openssl_methods
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
|
|
@ -34,6 +36,8 @@ class HTTP11DownloadHandler:
|
|||
lazy = False
|
||||
|
||||
def __init__(self, settings, crawler=None):
|
||||
self._crawler = crawler
|
||||
|
||||
from twisted.internet import reactor
|
||||
self._pool = HTTPConnectionPool(reactor, persistent=True)
|
||||
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
|
||||
|
|
@ -79,6 +83,7 @@ class HTTP11DownloadHandler:
|
|||
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
|
||||
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
|
||||
fail_on_dataloss=self._fail_on_dataloss,
|
||||
crawler=self._crawler,
|
||||
)
|
||||
return agent.download_request(request)
|
||||
|
||||
|
|
@ -276,7 +281,7 @@ class ScrapyAgent:
|
|||
_TunnelingAgent = TunnelingAgent
|
||||
|
||||
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
|
||||
maxsize=0, warnsize=0, fail_on_dataloss=True):
|
||||
maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
|
||||
self._contextFactory = contextFactory
|
||||
self._connectTimeout = connectTimeout
|
||||
self._bindAddress = bindAddress
|
||||
|
|
@ -285,6 +290,7 @@ class ScrapyAgent:
|
|||
self._warnsize = warnsize
|
||||
self._fail_on_dataloss = fail_on_dataloss
|
||||
self._txresponse = None
|
||||
self._crawler = crawler
|
||||
|
||||
def _get_agent(self, request, timeout):
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -407,7 +413,15 @@ class ScrapyAgent:
|
|||
|
||||
d = defer.Deferred(_cancel)
|
||||
txresponse.deliverBody(
|
||||
_ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
|
||||
_ResponseReader(
|
||||
finished=d,
|
||||
txresponse=txresponse,
|
||||
request=request,
|
||||
maxsize=maxsize,
|
||||
warnsize=warnsize,
|
||||
fail_on_dataloss=fail_on_dataloss,
|
||||
crawler=self._crawler,
|
||||
)
|
||||
)
|
||||
|
||||
# save response for timeouts
|
||||
|
|
@ -418,7 +432,7 @@ class ScrapyAgent:
|
|||
def _cb_bodydone(self, result, request, url):
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
return respcls(
|
||||
response = respcls(
|
||||
url=url,
|
||||
status=int(result["txresponse"].code),
|
||||
headers=headers,
|
||||
|
|
@ -427,6 +441,10 @@ class ScrapyAgent:
|
|||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
)
|
||||
if result.get("failure"):
|
||||
result["failure"].value.response = response
|
||||
return result["failure"]
|
||||
return response
|
||||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
|
|
@ -449,7 +467,7 @@ class _RequestBodyProducer:
|
|||
|
||||
class _ResponseReader(protocol.Protocol):
|
||||
|
||||
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
|
||||
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
|
||||
self._finished = finished
|
||||
self._txresponse = txresponse
|
||||
self._request = request
|
||||
|
|
@ -462,6 +480,17 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._bytes_received = 0
|
||||
self._certificate = None
|
||||
self._ip_address = None
|
||||
self._crawler = crawler
|
||||
|
||||
def _finish_response(self, flags=None, failure=None):
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": self._bodybuf.getvalue(),
|
||||
"flags": flags,
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
"failure": failure,
|
||||
})
|
||||
|
||||
def connectionMade(self):
|
||||
if self._certificate is None:
|
||||
|
|
@ -479,6 +508,20 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._bodybuf.write(bodyBytes)
|
||||
self._bytes_received += len(bodyBytes)
|
||||
|
||||
bytes_received_result = self._crawler.signals.send_catch_log(
|
||||
signal=signals.bytes_received,
|
||||
data=bodyBytes,
|
||||
request=self._request,
|
||||
spider=self._crawler.spider,
|
||||
)
|
||||
for handler, result in bytes_received_result:
|
||||
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
|
||||
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
|
||||
{"request": self._request, "handler": handler.__qualname__})
|
||||
self.transport._producer.loseConnection()
|
||||
failure = result if result.value.fail else None
|
||||
self._finish_response(flags=["download_stopped"], failure=failure)
|
||||
|
||||
if self._maxsize and self._bytes_received > self._maxsize:
|
||||
logger.error("Received (%(bytes)s) bytes larger than download "
|
||||
"max size (%(maxsize)s) in request %(request)s.",
|
||||
|
|
@ -500,36 +543,17 @@ class _ResponseReader(protocol.Protocol):
|
|||
if self._finished.called:
|
||||
return
|
||||
|
||||
body = self._bodybuf.getvalue()
|
||||
if reason.check(ResponseDone):
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": None,
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
self._finish_response()
|
||||
return
|
||||
|
||||
if reason.check(PotentialDataLoss):
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["partial"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
self._finish_response(flags=["partial"])
|
||||
return
|
||||
|
||||
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
|
||||
if not self._fail_on_dataloss:
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["dataloss"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
self._finish_response(flags=["dataloss"])
|
||||
return
|
||||
|
||||
elif not self._fail_on_dataloss_warned:
|
||||
|
|
|
|||
|
|
@ -100,11 +100,12 @@ class S3DownloadHandler:
|
|||
url=url, headers=awsrequest.headers.items())
|
||||
else:
|
||||
signed_headers = self.conn.make_request(
|
||||
method=request.method,
|
||||
bucket=bucket,
|
||||
key=unquote(p.path),
|
||||
query_args=unquote(p.query),
|
||||
headers=request.headers,
|
||||
data=request.body)
|
||||
method=request.method,
|
||||
bucket=bucket,
|
||||
key=unquote(p.path),
|
||||
query_args=unquote(p.query),
|
||||
headers=request.headers,
|
||||
data=request.body,
|
||||
)
|
||||
request = request.replace(url=url, headers=signed_headers)
|
||||
return self._download_http(request, spider)
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
self.transport.stopProducing()
|
||||
|
||||
self.factory.noPage(
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." %
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds."
|
||||
% (self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
|
|||
|
|
@ -217,11 +217,9 @@ class ExecutionEngine:
|
|||
self.slot.nextcall.schedule()
|
||||
|
||||
def schedule(self, request, spider):
|
||||
self.signals.send_catch_log(signal=signals.request_scheduled,
|
||||
request=request, spider=spider)
|
||||
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
|
||||
if not self.slot.scheduler.enqueue_request(request):
|
||||
self.signals.send_catch_log(signal=signals.request_dropped,
|
||||
request=request, spider=spider)
|
||||
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
|
||||
|
||||
def download(self, request, spider):
|
||||
d = self._download(request, spider)
|
||||
|
|
@ -247,8 +245,8 @@ class ExecutionEngine:
|
|||
logkws = self.logformatter.crawled(request, response, spider)
|
||||
if logkws is not None:
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
|
||||
self.signals.send_catch_log(signal=signals.response_received,
|
||||
response=response, request=request, spider=spider)
|
||||
self.signals.send_catch_log(signals.response_received,
|
||||
response=response, request=request, spider=spider)
|
||||
return response
|
||||
|
||||
def _on_complete(_):
|
||||
|
|
@ -286,8 +284,7 @@ class ExecutionEngine:
|
|||
next loop and this function is guaranteed to be called (at least) once
|
||||
again for this spider.
|
||||
"""
|
||||
res = self.signals.send_catch_log(signal=signals.spider_idle,
|
||||
spider=spider, dont_log=DontCloseSpider)
|
||||
res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
|
||||
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -4,18 +4,18 @@ extracts information from them"""
|
|||
import logging
|
||||
from collections import deque
|
||||
|
||||
from twisted.python.failure import Failure
|
||||
from itemadapter import is_item
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
|
||||
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
|
||||
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
|
||||
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
|
||||
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -191,7 +191,7 @@ class Scraper:
|
|||
"""
|
||||
if isinstance(output, Request):
|
||||
self.crawler.engine.crawl(request=output, spider=spider)
|
||||
elif isinstance(output, (BaseItem, dict)):
|
||||
elif is_item(output):
|
||||
self.slot.itemproc_size += 1
|
||||
dfd = self.itemproc.process_item(output, spider)
|
||||
dfd.addBoth(self._itemproc_finished, output, response, spider)
|
||||
|
|
@ -200,10 +200,11 @@ class Scraper:
|
|||
pass
|
||||
else:
|
||||
typename = type(output).__name__
|
||||
logger.error('Spider must return Request, BaseItem, dict or None, '
|
||||
'got %(typename)r in %(request)s',
|
||||
{'request': request, 'typename': typename},
|
||||
extra={'spider': spider})
|
||||
logger.error(
|
||||
'Spider must return request, item, or None, got %(typename)r in %(request)s',
|
||||
{'request': request, 'typename': typename},
|
||||
extra={'spider': spider},
|
||||
)
|
||||
|
||||
def _log_download_errors(self, spider_failure, download_failure, request, spider):
|
||||
"""Log and silence errors that come from the engine (typically download
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def _isiterable(possible_iterator):
|
|||
|
||||
|
||||
def _fname(f):
|
||||
return "%s.%s".format(
|
||||
return "{}.{}".format(
|
||||
f.__self__.__class__.__name__,
|
||||
f.__func__.__name__
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import re
|
||||
import logging
|
||||
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ class CookiesMiddleware:
|
|||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
for cookie in cookies:
|
||||
for cookie in self._get_request_cookies(jar, request):
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
|
||||
# set Cookie header
|
||||
|
|
@ -68,28 +67,65 @@ class CookiesMiddleware:
|
|||
msg = "Received cookies from: {}\n{}".format(response, cookies)
|
||||
logger.debug(msg, extra={'spider': spider})
|
||||
|
||||
def _format_cookie(self, cookie):
|
||||
# build cookie string
|
||||
cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
|
||||
|
||||
if cookie.get('path', None):
|
||||
cookie_str += '; Path=%s' % cookie['path']
|
||||
if cookie.get('domain', None):
|
||||
cookie_str += '; Domain=%s' % cookie['domain']
|
||||
def _format_cookie(self, cookie, request):
|
||||
"""
|
||||
Given a dict consisting of cookie components, return its string representation.
|
||||
Decode from bytes if necessary.
|
||||
"""
|
||||
decoded = {}
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
if not cookie.get(key):
|
||||
if key in ("name", "value"):
|
||||
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
|
||||
logger.warning(msg.format(request, cookie, key))
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], str):
|
||||
decoded[key] = cookie[key]
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request, cookie)
|
||||
decoded[key] = cookie[key].decode("latin1", errors="replace")
|
||||
|
||||
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
|
||||
for key, value in decoded.items(): # path, domain
|
||||
cookie_str += "; {}={}".format(key.capitalize(), value)
|
||||
return cookie_str
|
||||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
if isinstance(request.cookies, dict):
|
||||
cookie_list = [
|
||||
{'name': k, 'value': v}
|
||||
for k, v in request.cookies.items()
|
||||
]
|
||||
else:
|
||||
cookie_list = request.cookies
|
||||
"""
|
||||
Extract cookies from a Request. Values from the `Request.cookies` attribute
|
||||
take precedence over values from the `Cookie` request header.
|
||||
"""
|
||||
def get_cookies_from_header(jar, request):
|
||||
cookie_header = request.headers.get("Cookie")
|
||||
if not cookie_header:
|
||||
return []
|
||||
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
|
||||
cookie_list_unicode = []
|
||||
for cookie_bytes in cookie_gen_bytes:
|
||||
try:
|
||||
cookie_unicode = cookie_bytes.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request, cookie_bytes)
|
||||
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
|
||||
cookie_list_unicode.append(cookie_unicode)
|
||||
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
cookies = [self._format_cookie(x) for x in cookie_list]
|
||||
headers = {'Set-Cookie': cookies}
|
||||
response = Response(request.url, headers=headers)
|
||||
def get_cookies_from_attribute(jar, request):
|
||||
if not request.cookies:
|
||||
return []
|
||||
elif isinstance(request.cookies, dict):
|
||||
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
response = Response(request.url, headers={"Set-Cookie": formatted})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
return jar.make_cookies(response, request)
|
||||
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
|
||||
|
|
|
|||
|
|
@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages.
|
|||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import TimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
|
|||
|
|
@ -41,6 +41,18 @@ class CloseSpider(Exception):
|
|||
self.reason = reason
|
||||
|
||||
|
||||
class StopDownload(Exception):
|
||||
"""
|
||||
Stop the download of the body for a given response.
|
||||
The 'fail' boolean parameter indicates whether or not the resulting partial response
|
||||
should be handled by the request errback. Note that 'fail' is a keyword-only argument.
|
||||
"""
|
||||
|
||||
def __init__(self, *, fail=True):
|
||||
super().__init__()
|
||||
self.fail = fail
|
||||
|
||||
|
||||
# Items
|
||||
|
||||
|
||||
|
|
@ -59,6 +71,7 @@ class NotSupported(Exception):
|
|||
|
||||
class UsageError(Exception):
|
||||
"""To indicate a command-line usage error"""
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@ Item Exporters are used to export/serialize items into different formats.
|
|||
from collections import Mapping
|
||||
import csv
|
||||
import io
|
||||
import pprint
|
||||
import marshal
|
||||
import warnings
|
||||
import pickle
|
||||
import pprint
|
||||
import warnings
|
||||
from xml.sax.saxutils import XMLGenerator
|
||||
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
|
||||
from scrapy.item import BaseItem
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||
|
||||
|
||||
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
|
||||
|
|
@ -57,11 +59,14 @@ class BaseItemExporter:
|
|||
"""Return the fields to export as an iterable of tuples
|
||||
(name, serialized_value)
|
||||
"""
|
||||
item = ItemAdapter(item)
|
||||
|
||||
if include_empty is None:
|
||||
include_empty = self.export_empty_fields
|
||||
|
||||
if self.fields_to_export is None:
|
||||
if include_empty and not isinstance(item, dict):
|
||||
field_iter = item.fields.keys()
|
||||
if include_empty:
|
||||
field_iter = item.field_names()
|
||||
else:
|
||||
field_iter = item.keys()
|
||||
elif isinstance(self.fields_to_export, Mapping):
|
||||
|
|
@ -84,13 +89,8 @@ class BaseItemExporter:
|
|||
else:
|
||||
item_field, output_field = field_name
|
||||
if item_field in item:
|
||||
if isinstance(item, dict):
|
||||
field = {}
|
||||
else:
|
||||
field = item.fields[item_field]
|
||||
value = self.serialize_field(
|
||||
field, output_field, item[item_field]
|
||||
)
|
||||
field_meta = item.get_field_meta(item_field)
|
||||
value = self.serialize_field(field_meta, output_field, item[item_field])
|
||||
else:
|
||||
value = default_value
|
||||
|
||||
|
|
@ -272,7 +272,7 @@ class CsvItemExporter(BaseItemExporter):
|
|||
|
||||
class PickleItemExporter(BaseItemExporter):
|
||||
|
||||
def __init__(self, file, protocol=2, **kwargs):
|
||||
def __init__(self, file, protocol=4, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.file = file
|
||||
self.protocol = protocol
|
||||
|
|
@ -319,6 +319,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
|
||||
.. _msgpack: https://pypi.org/project/msgpack/
|
||||
"""
|
||||
|
||||
def _configure(self, options, dont_fail=False):
|
||||
self.binary = options.pop('binary', True)
|
||||
super(PythonItemExporter, self)._configure(options, dont_fail)
|
||||
|
|
@ -334,24 +335,24 @@ class PythonItemExporter(BaseItemExporter):
|
|||
return serializer(value)
|
||||
|
||||
def _serialize_value(self, value):
|
||||
if isinstance(value, BaseItem):
|
||||
if isinstance(value, _BaseItem):
|
||||
return self.export_item(value)
|
||||
if isinstance(value, dict):
|
||||
return dict(self._serialize_dict(value))
|
||||
if is_listlike(value):
|
||||
elif is_item(value):
|
||||
return dict(self._serialize_item(value))
|
||||
elif is_listlike(value):
|
||||
return [self._serialize_value(v) for v in value]
|
||||
encode_func = to_bytes if self.binary else to_unicode
|
||||
if isinstance(value, (str, bytes)):
|
||||
return encode_func(value, encoding=self.encoding)
|
||||
return value
|
||||
|
||||
def _serialize_dict(self, value):
|
||||
for key, val in value.items():
|
||||
def _serialize_item(self, item):
|
||||
for key, value in ItemAdapter(item).items():
|
||||
key = to_bytes(key) if self.binary else key
|
||||
yield key, self._serialize_value(val)
|
||||
yield key, self._serialize_value(value)
|
||||
|
||||
def export_item(self, item):
|
||||
result = dict(self._get_serialized_fields(item))
|
||||
if self.binary:
|
||||
result = dict(self._serialize_dict(result))
|
||||
result = dict(self._serialize_item(result))
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -270,7 +270,9 @@ class FeedExporter:
|
|||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
deferred_list.append(d)
|
||||
continue
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
|
|
|
|||
|
|
@ -46,9 +46,10 @@ class RFC2616Policy:
|
|||
def __init__(self, settings):
|
||||
self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE')
|
||||
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
|
||||
self.ignore_response_cache_controls = [to_bytes(cc) for cc in
|
||||
settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')]
|
||||
self._cc_parsed = WeakKeyDictionary()
|
||||
self.ignore_response_cache_controls = [
|
||||
to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')
|
||||
]
|
||||
|
||||
def _parse_cachecontrol(self, r):
|
||||
if r not in self._cc_parsed:
|
||||
|
|
@ -250,7 +251,7 @@ class DbmCacheStorage:
|
|||
'headers': dict(response.headers),
|
||||
'body': response.body,
|
||||
}
|
||||
self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
|
||||
self.db['%s_data' % key] = pickle.dumps(data, protocol=4)
|
||||
self.db['%s_time' % key] = str(time())
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
|
|
@ -317,7 +318,7 @@ class FilesystemCacheStorage:
|
|||
with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
|
||||
f.write(to_bytes(repr(metadata)))
|
||||
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
|
||||
pickle.dump(metadata, f, protocol=2)
|
||||
pickle.dump(metadata, f, protocol=4)
|
||||
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
|
||||
f.write(headers_dict_to_raw(response.headers))
|
||||
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class SpiderState:
|
|||
def spider_closed(self, spider):
|
||||
if self.jobdir:
|
||||
with open(self.statefn, 'wb') as f:
|
||||
pickle.dump(spider.state, f, protocol=2)
|
||||
pickle.dump(spider.state, f, protocol=4)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
if self.jobdir and os.path.exists(self.statefn):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ discovering (through HTTP headers) to base Response class.
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import json
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from typing import Generator
|
||||
from urllib.parse import urljoin
|
||||
|
|
@ -14,15 +16,19 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
|
|||
http_content_type_encoding, resolve_encoding)
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
|
||||
_NONE = object()
|
||||
|
||||
|
||||
class TextResponse(Response):
|
||||
|
||||
_DEFAULT_ENCODING = 'ascii'
|
||||
_cached_decoded_json = _NONE
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._encoding = kwargs.pop('encoding', None)
|
||||
|
|
@ -61,8 +67,19 @@ class TextResponse(Response):
|
|||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
warnings.warn('Response.body_as_unicode() is deprecated, '
|
||||
'please use Response.text instead.',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return self.text
|
||||
|
||||
def json(self):
|
||||
"""
|
||||
Deserialize a JSON document to a Python object.
|
||||
"""
|
||||
if self._cached_decoded_json is _NONE:
|
||||
self._cached_decoded_json = json.loads(self.text)
|
||||
return self._cached_decoded_json
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
""" Body as unicode """
|
||||
|
|
|
|||
|
|
@ -14,28 +14,39 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
|||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
class BaseItem(object_ref):
|
||||
"""Base class for all scraped items.
|
||||
|
||||
In Scrapy, an object is considered an *item* if it is an instance of either
|
||||
:class:`BaseItem` or :class:`dict`. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`BaseItem` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`BaseItem` or :class:`dict`.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
class _BaseItem(object_ref):
|
||||
"""
|
||||
Temporary class used internally to avoid the deprecation
|
||||
warning raised by isinstance checks using BaseItem.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class _BaseItemMeta(ABCMeta):
|
||||
def __instancecheck__(cls, instance):
|
||||
if cls is BaseItem:
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super().__instancecheck__(instance)
|
||||
|
||||
|
||||
class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
||||
"""
|
||||
Deprecated, please use :class:`scrapy.item.Item` instead
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
"""Container of field metadata"""
|
||||
|
||||
|
||||
class ItemMeta(ABCMeta):
|
||||
class ItemMeta(_BaseItemMeta):
|
||||
"""Metaclass_ of :class:`Item` that handles field definitions.
|
||||
|
||||
.. _metaclass: https://realpython.com/python-metaclasses
|
||||
|
|
@ -68,8 +79,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use '
|
||||
'scrapy.item.Item instead',
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(DictItem, cls).__new__(cls, *args, **kwargs)
|
||||
|
||||
|
|
@ -86,8 +96,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
if key in self.fields:
|
||||
self._values[key] = value
|
||||
else:
|
||||
raise KeyError("%s does not support field: %s" %
|
||||
(self.__class__.__name__, key))
|
||||
raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key))
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self._values[key]
|
||||
|
|
@ -99,8 +108,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
|
||||
def __setattr__(self, name, value):
|
||||
if not name.startswith('_'):
|
||||
raise AttributeError("Use item[%r] = %r to set field value" %
|
||||
(name, value))
|
||||
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
|
||||
super(DictItem, self).__setattr__(name, value)
|
||||
|
||||
def __len__(self):
|
||||
|
|
@ -127,4 +135,24 @@ class DictItem(MutableMapping, BaseItem):
|
|||
|
||||
|
||||
class Item(DictItem, metaclass=ItemMeta):
|
||||
pass
|
||||
"""
|
||||
Base class for scraped items.
|
||||
|
||||
In Scrapy, an object is considered an ``item`` if it is an instance of either
|
||||
:class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
|
||||
spider callback is evaluated, only instances of :class:`Item` or
|
||||
:class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
If you need instances of a custom class to be considered items by Scrapy,
|
||||
you must inherit from either :class:`Item` or :class:`dict`.
|
||||
|
||||
Items must declare :class:`Field` attributes, which are processed and stored
|
||||
in the ``fields`` attribute. This restricts the set of allowed field names
|
||||
and prevents typos, raising ``KeyError`` when referring to undefined fields.
|
||||
Additionally, fields can be used to define metadata and control the way
|
||||
data is processed internally. Please refer to the :ref:`documentation
|
||||
about fields <topics-items-fields>` for additional information.
|
||||
|
||||
Unlike instances of :class:`dict`, instances of :class:`Item` may be
|
||||
:ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -133,4 +133,4 @@ class FilteringLinkExtractor:
|
|||
|
||||
|
||||
# Top-level imports
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
|
||||
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
"""
|
||||
Link extractor based on lxml.html
|
||||
"""
|
||||
import operator
|
||||
from functools import partial
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import lxml.etree as etree
|
||||
|
|
@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace
|
|||
from w3lib.url import canonicalize_url, safe_url_string
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
|
||||
|
||||
# from lxml/src/lxml/html/__init__.py
|
||||
|
|
@ -27,19 +29,24 @@ def _nons(tag):
|
|||
return tag
|
||||
|
||||
|
||||
def _identity(x):
|
||||
return x
|
||||
|
||||
|
||||
def _canonicalize_link_url(link):
|
||||
return canonicalize_url(link.url, keep_fragments=True)
|
||||
|
||||
|
||||
class LxmlParserLinkExtractor:
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True, canonicalized=False):
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
def __init__(
|
||||
self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False
|
||||
):
|
||||
self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
|
||||
self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
|
||||
self.process_attr = process if callable(process) else _identity
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
if canonicalized:
|
||||
self.link_key = lambda link: link.url
|
||||
else:
|
||||
self.link_key = lambda link: canonicalize_url(link.url,
|
||||
keep_fragments=True)
|
||||
self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url
|
||||
|
||||
def _iter_links(self, document):
|
||||
for el in document.iter(etree.Element):
|
||||
|
|
@ -93,25 +100,44 @@ class LxmlParserLinkExtractor:
|
|||
|
||||
class LxmlLinkExtractor(FilteringLinkExtractor):
|
||||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href',), canonicalize=False,
|
||||
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True, restrict_text=None):
|
||||
def __init__(
|
||||
self,
|
||||
allow=(),
|
||||
deny=(),
|
||||
allow_domains=(),
|
||||
deny_domains=(),
|
||||
restrict_xpaths=(),
|
||||
tags=('a', 'area'),
|
||||
attrs=('href',),
|
||||
canonicalize=False,
|
||||
unique=True,
|
||||
process_value=None,
|
||||
deny_extensions=None,
|
||||
restrict_css=(),
|
||||
strip=True,
|
||||
restrict_text=None,
|
||||
):
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
lx = LxmlParserLinkExtractor(
|
||||
tag=lambda x: x in tags,
|
||||
attr=lambda x: x in attrs,
|
||||
tag=partial(operator.contains, tags),
|
||||
attr=partial(operator.contains, attrs),
|
||||
unique=unique,
|
||||
process=process_value,
|
||||
strip=strip,
|
||||
canonicalized=canonicalize
|
||||
)
|
||||
|
||||
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
|
||||
allow_domains=allow_domains, deny_domains=deny_domains,
|
||||
restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
|
||||
canonicalize=canonicalize, deny_extensions=deny_extensions,
|
||||
restrict_text=restrict_text)
|
||||
super(LxmlLinkExtractor, self).__init__(
|
||||
link_extractor=lx,
|
||||
allow=allow,
|
||||
deny=deny,
|
||||
allow_domains=allow_domains,
|
||||
deny_domains=deny_domains,
|
||||
restrict_xpaths=restrict_xpaths,
|
||||
restrict_css=restrict_css,
|
||||
canonicalize=canonicalize,
|
||||
deny_extensions=deny_extensions,
|
||||
restrict_text=restrict_text,
|
||||
)
|
||||
|
||||
def extract_links(self, response):
|
||||
"""Returns a list of :class:`~scrapy.link.Link` objects from the
|
||||
|
|
@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
"""
|
||||
base_url = get_base_url(response)
|
||||
if self.restrict_xpaths:
|
||||
docs = [subdoc
|
||||
for x in self.restrict_xpaths
|
||||
for subdoc in response.xpath(x)]
|
||||
docs = [
|
||||
subdoc
|
||||
for x in self.restrict_xpaths
|
||||
for subdoc in response.xpath(x)
|
||||
]
|
||||
else:
|
||||
docs = [response.selector]
|
||||
all_links = []
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ See documentation in docs/topics/loaders.rst
|
|||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.loader.processors import Identity
|
||||
|
|
@ -44,7 +46,7 @@ class ItemLoader:
|
|||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
# values from initial item
|
||||
for field_name, value in item.items():
|
||||
for field_name, value in ItemAdapter(item).items():
|
||||
self._values[field_name] += arg_to_iter(value)
|
||||
|
||||
@property
|
||||
|
|
@ -127,13 +129,12 @@ class ItemLoader:
|
|||
return value
|
||||
|
||||
def load_item(self):
|
||||
item = self.item
|
||||
adapter = ItemAdapter(self.item)
|
||||
for field_name in tuple(self._values):
|
||||
value = self.get_output_value(field_name)
|
||||
if value is not None:
|
||||
item[field_name] = value
|
||||
|
||||
return item
|
||||
adapter[field_name] = value
|
||||
return adapter.item
|
||||
|
||||
def get_output_value(self, field_name):
|
||||
proc = self.get_output_processor(field_name)
|
||||
|
|
@ -174,11 +175,8 @@ class ItemLoader:
|
|||
value, type(e).__name__, str(e)))
|
||||
|
||||
def _get_item_field_attr(self, field_name, key, default=None):
|
||||
if isinstance(self.item, Item):
|
||||
value = self.item.fields[field_name].get(key, default)
|
||||
else:
|
||||
value = default
|
||||
return value
|
||||
field_meta = ItemAdapter(self.item).get_field_meta(field_name)
|
||||
return field_meta.get(key, default)
|
||||
|
||||
def _check_selector_method(self):
|
||||
if self.selector is None:
|
||||
|
|
|
|||
|
|
@ -28,8 +28,10 @@ def _to_bytes_or_none(text):
|
|||
|
||||
|
||||
class MailSender:
|
||||
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
|
||||
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
|
||||
def __init__(
|
||||
self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None,
|
||||
smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False
|
||||
):
|
||||
self.smtphost = smtphost
|
||||
self.smtpport = smtpport
|
||||
self.smtpuser = _to_bytes_or_none(smtpuser)
|
||||
|
|
@ -41,9 +43,15 @@ class MailSender:
|
|||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'],
|
||||
settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
|
||||
settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
|
||||
return cls(
|
||||
smtphost=settings['MAIL_HOST'],
|
||||
mailfrom=settings['MAIL_FROM'],
|
||||
smtpuser=settings['MAIL_USER'],
|
||||
smtppass=settings['MAIL_PASS'],
|
||||
smtpport=settings.getint('MAIL_PORT'),
|
||||
smtptls=settings.getbool('MAIL_TLS'),
|
||||
smtpssl=settings.getbool('MAIL_SSL'),
|
||||
)
|
||||
|
||||
def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -89,9 +97,12 @@ class MailSender:
|
|||
return
|
||||
|
||||
dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
|
||||
dfd.addCallbacks(self._sent_ok, self._sent_failed,
|
||||
dfd.addCallbacks(
|
||||
callback=self._sent_ok,
|
||||
errback=self._sent_failed,
|
||||
callbackArgs=[to, cc, subject, len(attachs)],
|
||||
errbackArgs=[to, cc, subject, len(attachs)])
|
||||
errbackArgs=[to, cc, subject, len(attachs)],
|
||||
)
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd)
|
||||
return dfd
|
||||
|
||||
|
|
@ -115,9 +126,10 @@ class MailSender:
|
|||
from twisted.mail.smtp import ESMTPSenderFactory
|
||||
msg = BytesIO(msg)
|
||||
d = defer.Deferred()
|
||||
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
|
||||
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
|
||||
requireTransportSecurity=self.smtptls)
|
||||
factory = ESMTPSenderFactory(
|
||||
self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
|
||||
heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
|
||||
)
|
||||
factory.noisy = False
|
||||
|
||||
if self.smtpssl:
|
||||
|
|
|
|||
|
|
@ -10,24 +10,26 @@ import mimetypes
|
|||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from email.utils import parsedate_tz, mktime_tz
|
||||
from contextlib import suppress
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from ftplib import FTP
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Request
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.misc import md5sum
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.request import referer_str
|
||||
from scrapy.utils.boto import is_botocore
|
||||
from scrapy.utils.datatypes import CaselessDict
|
||||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import md5sum
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.request import referer_str
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -83,8 +85,7 @@ class S3FilesStore:
|
|||
AWS_USE_SSL = None
|
||||
AWS_VERIFY = None
|
||||
|
||||
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in
|
||||
# FilesPipeline.from_settings.
|
||||
POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
|
||||
HEADERS = {
|
||||
'Cache-Control': 'max-age=172800',
|
||||
}
|
||||
|
|
@ -433,7 +434,7 @@ class FilesPipeline(MediaPipeline):
|
|||
self.inc_stats(info.spider, 'uptodate')
|
||||
|
||||
checksum = result.get('checksum', None)
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum}
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
|
||||
|
||||
path = self.file_path(request, info=info)
|
||||
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
|
||||
|
|
@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline):
|
|||
)
|
||||
raise FileException(str(exc))
|
||||
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum}
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}
|
||||
|
||||
def inc_stats(self, spider, status):
|
||||
spider.crawler.stats.inc_value('file_count', spider=spider)
|
||||
|
|
@ -518,7 +519,8 @@ class FilesPipeline(MediaPipeline):
|
|||
|
||||
# Overridable Interface
|
||||
def get_media_requests(self, item, info):
|
||||
return [Request(x) for x in item.get(self.files_urls_field, [])]
|
||||
urls = ItemAdapter(item).get(self.files_urls_field, [])
|
||||
return [Request(u) for u in urls]
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
|
|
@ -529,8 +531,8 @@ class FilesPipeline(MediaPipeline):
|
|||
return checksum
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
if isinstance(item, dict) or self.files_result_field in item.fields:
|
||||
item[self.files_result_field] = [x for ok, x in results if ok]
|
||||
with suppress(KeyError):
|
||||
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
|
|
|
|||
|
|
@ -5,17 +5,19 @@ See documentation in topics/media-pipeline.rst
|
|||
"""
|
||||
import functools
|
||||
import hashlib
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
from PIL import Image
|
||||
|
||||
from scrapy.exceptions import DropItem
|
||||
from scrapy.http import Request
|
||||
from scrapy.pipelines.files import FileException, FilesPipeline
|
||||
# TODO: from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.misc import md5sum
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.http import Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.exceptions import DropItem
|
||||
# TODO: from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.files import FileException, FilesPipeline
|
||||
|
||||
|
||||
class NoimagesDrop(DropItem):
|
||||
|
|
@ -157,11 +159,12 @@ class ImagesPipeline(FilesPipeline):
|
|||
return image, buf
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
return [Request(x) for x in item.get(self.images_urls_field, [])]
|
||||
urls = ItemAdapter(item).get(self.images_urls_field, [])
|
||||
return [Request(u) for u in urls]
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
if isinstance(item, dict) or self.images_result_field in item.fields:
|
||||
item[self.images_result_field] = [x for ok, x in results if ok]
|
||||
with suppress(KeyError):
|
||||
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
|
|
|
|||
|
|
@ -58,9 +58,9 @@ class ResponseTypes:
|
|||
|
||||
def from_content_disposition(self, content_disposition):
|
||||
try:
|
||||
filename = to_unicode(content_disposition,
|
||||
encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
|
||||
filename = filename.strip('"\'')
|
||||
filename = to_unicode(
|
||||
content_disposition, encoding='latin-1', errors='replace'
|
||||
).split(';')[1].split('=')[1].strip('"\'')
|
||||
return self.from_filename(filename)
|
||||
except IndexError:
|
||||
return Response
|
||||
|
|
|
|||
|
|
@ -17,10 +17,12 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
|
|||
except UnicodeDecodeError:
|
||||
# If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
|
||||
# Switch to 'allow all' state.
|
||||
logger.warning("Failure while parsing robots.txt. "
|
||||
"File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
|
||||
exc_info=sys.exc_info(),
|
||||
extra={'spider': spider})
|
||||
logger.warning(
|
||||
"Failure while parsing robots.txt. File either contains garbage or "
|
||||
"is in an encoding other than UTF-8, treating it as an empty file.",
|
||||
exc_info=sys.exc_info(),
|
||||
extra={'spider': spider},
|
||||
)
|
||||
robotstxt_body = ''
|
||||
return robotstxt_body
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
"""
|
||||
Selectors
|
||||
"""
|
||||
from scrapy.selector.unified import * # noqa: F401
|
||||
|
||||
# top-level imports
|
||||
from scrapy.selector.unified import Selector, SelectorList
|
||||
|
|
|
|||
|
|
@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref):
|
|||
selectorlist_cls = SelectorList
|
||||
|
||||
def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
|
||||
if not(response is None or text is None):
|
||||
raise ValueError('%s.__init__() received both response and text'
|
||||
% self.__class__.__name__)
|
||||
if response is not None and text is not None:
|
||||
raise ValueError('%s.__init__() received both response and text'
|
||||
% self.__class__.__name__)
|
||||
|
||||
st = _st(response, type or self._default_type)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
|
|||
import os
|
||||
import signal
|
||||
|
||||
from itemadapter import is_item
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python import threadable
|
||||
from w3lib.url import any_to_uri
|
||||
|
|
@ -13,21 +14,18 @@ from w3lib.url import any_to_uri
|
|||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.console import start_python_console
|
||||
from scrapy.utils.conf import get_config
|
||||
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.response import open_in_browser
|
||||
from scrapy.utils.conf import get_config
|
||||
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
|
||||
|
||||
|
||||
class Shell:
|
||||
|
||||
relevant_classes = (Crawler, Spider, Request, Response, BaseItem,
|
||||
Settings)
|
||||
relevant_classes = (Crawler, Spider, Request, Response, Settings)
|
||||
|
||||
def __init__(self, crawler, update_vars=None, code=None):
|
||||
self.crawler = crawler
|
||||
|
|
@ -146,17 +144,16 @@ class Shell:
|
|||
b.append("Useful shortcuts:")
|
||||
if self.inthread:
|
||||
b.append(" fetch(url[, redirect=True]) "
|
||||
"Fetch URL and update local objects "
|
||||
"(by default, redirects are followed)")
|
||||
"Fetch URL and update local objects (by default, redirects are followed)")
|
||||
b.append(" fetch(req) "
|
||||
"Fetch a scrapy.Request and update local objects ")
|
||||
b.append(" shelp() Shell help (print this help)")
|
||||
b.append(" view(response) View response in a browser")
|
||||
|
||||
return "\n".join("[s] %s" % l for l in b)
|
||||
return "\n".join("[s] %s" % line for line in b)
|
||||
|
||||
def _is_relevant(self, value):
|
||||
return isinstance(value, self.relevant_classes)
|
||||
return isinstance(value, self.relevant_classes) or is_item(value)
|
||||
|
||||
|
||||
def inspect_response(response, spider):
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ request_reached_downloader = object()
|
|||
request_left_downloader = object()
|
||||
response_received = object()
|
||||
response_downloaded = object()
|
||||
bytes_received = object()
|
||||
item_scraped = object()
|
||||
item_dropped = object()
|
||||
item_error = object()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from collections import defaultdict
|
||||
import traceback
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
|
||||
from zope.interface import implementer
|
||||
|
||||
|
|
@ -16,6 +15,7 @@ class SpiderLoader:
|
|||
SpiderLoader is a class which locates and loads spiders
|
||||
in a Scrapy project.
|
||||
"""
|
||||
|
||||
def __init__(self, settings):
|
||||
self.spider_modules = settings.getlist('SPIDER_MODULES')
|
||||
self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
|
||||
|
|
@ -24,16 +24,21 @@ class SpiderLoader:
|
|||
self._load_all_spiders()
|
||||
|
||||
def _check_name_duplicates(self):
|
||||
dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(
|
||||
module=mod, cls=cls, name=name)
|
||||
for (mod, cls) in locations)
|
||||
for name, locations in self._found.items()
|
||||
if len(locations) > 1]
|
||||
dupes = []
|
||||
for name, locations in self._found.items():
|
||||
dupes.extend([
|
||||
" {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name)
|
||||
for mod, cls in locations
|
||||
if len(locations) > 1
|
||||
])
|
||||
|
||||
if dupes:
|
||||
msg = ("There are several spiders with the same name:\n\n"
|
||||
"{}\n\n This can cause unexpected behavior.".format(
|
||||
"\n\n".join(dupes)))
|
||||
warnings.warn(msg, UserWarning)
|
||||
dupes_string = "\n\n".join(dupes)
|
||||
warnings.warn(
|
||||
"There are several spiders with the same name:\n\n"
|
||||
"{}\n\n This can cause unexpected behavior.".format(dupes_string),
|
||||
category=UserWarning,
|
||||
)
|
||||
|
||||
def _load_spiders(self, module):
|
||||
for spcls in iter_spider_classes(module):
|
||||
|
|
@ -45,12 +50,15 @@ class SpiderLoader:
|
|||
try:
|
||||
for module in walk_modules(name):
|
||||
self._load_spiders(module)
|
||||
except ImportError as e:
|
||||
except ImportError:
|
||||
if self.warn_only:
|
||||
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
|
||||
"See above traceback for details.".format(
|
||||
modname=name, tb=traceback.format_exc()))
|
||||
warnings.warn(msg, RuntimeWarning)
|
||||
warnings.warn(
|
||||
"\n{tb}Could not load spiders from module '{modname}'. "
|
||||
"See above traceback for details.".format(
|
||||
modname=name, tb=traceback.format_exc()
|
||||
),
|
||||
category=RuntimeWarning,
|
||||
)
|
||||
else:
|
||||
raise
|
||||
self._check_name_duplicates()
|
||||
|
|
@ -73,8 +81,10 @@ class SpiderLoader:
|
|||
"""
|
||||
Return the list of spider names that can handle the given request.
|
||||
"""
|
||||
return [name for name, cls in self._spiders.items()
|
||||
if cls.handles_request(request)]
|
||||
return [
|
||||
name for name, cls in self._spiders.items()
|
||||
if cls.handles_request(request)
|
||||
]
|
||||
|
||||
def list(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -110,6 +110,6 @@ class Spider(object_ref):
|
|||
|
||||
|
||||
# Top-level imports
|
||||
from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401
|
||||
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401
|
||||
from scrapy.spiders.sitemap import SitemapSpider # noqa: F401
|
||||
from scrapy.spiders.crawl import CrawlSpider, Rule
|
||||
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
|
||||
from scrapy.spiders.sitemap import SitemapSpider
|
||||
|
|
|
|||
|
|
@ -54,8 +54,12 @@ class Rule:
|
|||
self.process_request = _get_method(self.process_request, spider)
|
||||
self.process_request_argcount = len(get_func_args(self.process_request))
|
||||
if self.process_request_argcount == 1:
|
||||
msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
|
||||
warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
warnings.warn(
|
||||
"Rule.process_request should accept two arguments "
|
||||
"(request, response), accepting only one is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
def _process_request(self, request, response):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class XMLFeedSpider(Spider):
|
|||
processing required before returning the results to the framework core,
|
||||
for example setting the item GUIDs. It receives a list of results and
|
||||
the response which originated that results. It must return a list of
|
||||
results (Items or Requests).
|
||||
results (items or requests).
|
||||
"""
|
||||
return results
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ class XMLFeedSpider(Spider):
|
|||
"""This method is called for the nodes matching the provided tag name
|
||||
(itertag). Receives the response and an Selector for each node.
|
||||
Overriding this method is mandatory. Otherwise, you spider won't work.
|
||||
This method must return either a BaseItem, a Request, or a list
|
||||
This method must return either an item, a request, or a list
|
||||
containing any of them.
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -96,5 +96,4 @@ def iterloc(it, alt=False):
|
|||
|
||||
# Also consider alternate URLs (xhtml:link rel="alternate")
|
||||
if alt and 'alternate' in d:
|
||||
for l in d['alternate']:
|
||||
yield l
|
||||
yield from d['alternate']
|
||||
|
|
|
|||
|
|
@ -81,12 +81,11 @@ def _scrapy_non_serialization_queue(queue_class):
|
|||
|
||||
def _pickle_serialize(obj):
|
||||
try:
|
||||
return pickle.dumps(obj, protocol=2)
|
||||
# Python <= 3.4 raises pickle.PicklingError here while
|
||||
# 3.5 <= Python < 3.6 raises AttributeError and
|
||||
# Python >= 3.6 raises TypeError
|
||||
return pickle.dumps(obj, protocol=4)
|
||||
# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
|
||||
# TypeError is raised from parsel.Selector
|
||||
except (pickle.PicklingError, AttributeError, TypeError) as e:
|
||||
raise ValueError(str(e))
|
||||
raise ValueError(str(e)) from e
|
||||
|
||||
|
||||
PickleFifoDiskQueueNonRequest = _serializable_queue(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your scraped items
|
||||
#
|
||||
# See documentation in:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define here the models for your spider middleware
|
||||
#
|
||||
# See documentation in:
|
||||
|
|
@ -7,6 +5,9 @@
|
|||
|
||||
from scrapy import signals
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
|
||||
|
||||
class ${ProjectName}SpiderMiddleware:
|
||||
# Not all methods need to be defined. If a method is not defined,
|
||||
|
|
@ -31,7 +32,7 @@ class ${ProjectName}SpiderMiddleware:
|
|||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
# Must return an iterable of Request, dict or Item objects.
|
||||
# Must return an iterable of Request, or item objects.
|
||||
for i in result:
|
||||
yield i
|
||||
|
||||
|
|
@ -39,8 +40,7 @@ class ${ProjectName}SpiderMiddleware:
|
|||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
# Should return either None or an iterable of Request, dict
|
||||
# or Item objects.
|
||||
# Should return either None or an iterable of Request or item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Define your item pipelines here
|
||||
#
|
||||
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
|
||||
|
||||
# useful for handling different item types with a single interface
|
||||
from itemadapter import ItemAdapter
|
||||
|
||||
|
||||
class ${ProjectName}Pipeline:
|
||||
def process_item(self, item, spider):
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Scrapy settings for $project_name project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import scrapy
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import scrapy
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import CrawlSpider, Rule
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from scrapy.spiders import CSVFeedSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from scrapy.spiders import XMLFeedSpider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
|
|||
def _embed_bpython_shell(namespace={}, banner=''):
|
||||
"""Start a bpython shell"""
|
||||
import bpython
|
||||
|
||||
@wraps(_embed_bpython_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
bpython.embed(locals_=namespace, banner=banner)
|
||||
|
|
@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
|
|||
def _embed_ptpython_shell(namespace={}, banner=''):
|
||||
"""Start a ptpython shell"""
|
||||
import ptpython.repl
|
||||
|
||||
@wraps(_embed_ptpython_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
print(banner)
|
||||
|
|
|
|||
|
|
@ -105,8 +105,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
|||
def __getitem__(self, key):
|
||||
try:
|
||||
return super(LocalWeakReferencedCache, self).__getitem__(key)
|
||||
except TypeError:
|
||||
return None # key is not weak-referenceable, it's not cached
|
||||
except (TypeError, KeyError):
|
||||
return None # key is either not weak-referenceable or not cached
|
||||
|
||||
|
||||
class SequenceExclude:
|
||||
|
|
|
|||
|
|
@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
|
|||
"""Return a Deferred built by chaining the given callbacks and errbacks"""
|
||||
d = defer.Deferred()
|
||||
for cb, eb in zip(callbacks, errbacks):
|
||||
d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw,
|
||||
errbackArgs=a, errbackKeywords=kw)
|
||||
d.addCallbacks(
|
||||
callback=cb, errback=eb,
|
||||
callbackArgs=a, callbackKeywords=kw,
|
||||
errbackArgs=a, errbackKeywords=kw,
|
||||
)
|
||||
if isinstance(input, failure.Failure):
|
||||
d.errback(input)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import warnings
|
||||
|
|
@ -39,7 +37,7 @@ class TopLevelFormatter(logging.Filter):
|
|||
self.loggers = loggers or []
|
||||
|
||||
def filter(self, record):
|
||||
if any(record.name.startswith(l + '.') for l in self.loggers):
|
||||
if any(record.name.startswith(logger + '.') for logger in self.loggers):
|
||||
record.name = record.name.split('.', 1)[0]
|
||||
return True
|
||||
|
||||
|
|
@ -144,10 +142,12 @@ def _get_handler(settings):
|
|||
def log_scrapy_info(settings):
|
||||
logger.info("Scrapy %(version)s started (bot: %(bot)s)",
|
||||
{'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
|
||||
logger.info("Versions: %(versions)s",
|
||||
{'versions': ", ".join("%s %s" % (name, version)
|
||||
for name, version in scrapy_components_versions()
|
||||
if name != "Scrapy")})
|
||||
versions = [
|
||||
"%s %s" % (name, version)
|
||||
for name, version in scrapy_components_versions()
|
||||
if name != "Scrapy"
|
||||
]
|
||||
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
|
||||
from twisted.internet import reactor
|
||||
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ from w3lib.html import replace_entities
|
|||
|
||||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.item import _BaseItem
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
|
||||
|
||||
def arg_to_iter(arg):
|
||||
|
|
@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
``*args`` and ``**kwargs`` are forwarded to the constructors.
|
||||
|
||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
||||
|
||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
"""
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
raise ValueError("Specify at least one of settings and crawler.")
|
||||
settings = crawler.settings
|
||||
if crawler and hasattr(objcls, 'from_crawler'):
|
||||
return objcls.from_crawler(crawler, *args, **kwargs)
|
||||
instance = objcls.from_crawler(crawler, *args, **kwargs)
|
||||
method_name = 'from_crawler'
|
||||
elif hasattr(objcls, 'from_settings'):
|
||||
return objcls.from_settings(settings, *args, **kwargs)
|
||||
instance = objcls.from_settings(settings, *args, **kwargs)
|
||||
method_name = 'from_settings'
|
||||
else:
|
||||
return objcls(*args, **kwargs)
|
||||
instance = objcls(*args, **kwargs)
|
||||
method_name = '__new__'
|
||||
if instance is None:
|
||||
raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
|
||||
return instance
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ def memoizemethod_noargs(method):
|
|||
weak reference to its object
|
||||
"""
|
||||
cache = weakref.WeakKeyDictionary()
|
||||
|
||||
@wraps(method)
|
||||
def new_method(self, *args, **kwargs):
|
||||
if self not in cache:
|
||||
|
|
@ -333,10 +334,10 @@ class MutableChain:
|
|||
"""
|
||||
|
||||
def __init__(self, *args):
|
||||
self.data = chain(*args)
|
||||
self.data = chain.from_iterable(args)
|
||||
|
||||
def extend(self, *iterables):
|
||||
self.data = chain(self.data, *iterables)
|
||||
self.data = chain(self.data, chain.from_iterable(iterables))
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -19,8 +19,7 @@ def get_base_url(response):
|
|||
"""Return the base url of the given response, joined with the response url"""
|
||||
if response not in _baseurl_cache:
|
||||
text = response.text[0:4096]
|
||||
_baseurl_cache[response] = html.get_base_url(text, response.url,
|
||||
response.encoding)
|
||||
_baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding)
|
||||
return _baseurl_cache[response]
|
||||
|
||||
|
||||
|
|
@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
|
|||
"""Parse the http-equiv refrsh parameter from the given response"""
|
||||
if response not in _metaref_cache:
|
||||
text = response.text[0:4096]
|
||||
_metaref_cache[response] = html.get_meta_refresh(text, response.url,
|
||||
response.encoding, ignore_tags=ignore_tags)
|
||||
_metaref_cache[response] = html.get_meta_refresh(
|
||||
text, response.url, response.encoding, ignore_tags=ignore_tags)
|
||||
return _metaref_cache[response]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ import json
|
|||
import datetime
|
||||
import decimal
|
||||
|
||||
from itemadapter import is_item, ItemAdapter
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import BaseItem
|
||||
|
||||
|
||||
class ScrapyJSONEncoder(json.JSONEncoder):
|
||||
|
|
@ -26,8 +26,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
|
|||
return str(o)
|
||||
elif isinstance(o, defer.Deferred):
|
||||
return str(o)
|
||||
elif isinstance(o, BaseItem):
|
||||
return dict(o)
|
||||
elif is_item(o):
|
||||
return ItemAdapter(o).asdict()
|
||||
elif isinstance(o, Request):
|
||||
return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
|
||||
elif isinstance(o, Response):
|
||||
|
|
|
|||
|
|
@ -5,13 +5,14 @@ import logging
|
|||
from twisted.internet.defer import DeferredList, Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
|
||||
getAllReceivers, disconnect
|
||||
from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
|
||||
from pydispatch.robustapply import robustApply
|
||||
|
||||
from scrapy.exceptions import StopDownload
|
||||
from scrapy.utils.defer import maybeDeferred_coro
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -23,13 +24,12 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
|
|||
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
|
||||
Failures instead of exceptions.
|
||||
"""
|
||||
dont_log = named.pop('dont_log', _IgnoredException)
|
||||
dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
|
||||
spider = named.get('spider', None)
|
||||
responses = []
|
||||
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
||||
try:
|
||||
response = robustApply(receiver, signal=signal, sender=sender,
|
||||
*arguments, **named)
|
||||
response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
|
||||
if isinstance(response, Deferred):
|
||||
logger.error("Cannot return deferreds from signal handler: %(receiver)s",
|
||||
{'receiver': receiver}, extra={'spider': spider})
|
||||
|
|
@ -63,8 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
|
|||
spider = named.get('spider', None)
|
||||
dfds = []
|
||||
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
||||
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
|
||||
*arguments, **named)
|
||||
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
|
||||
d.addErrback(logerror, receiver)
|
||||
d.addBoth(lambda result: (receiver, result))
|
||||
dfds.append(d)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
import inspect
|
||||
import logging
|
||||
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
|
|
@ -18,7 +18,11 @@ def iterate_spider_output(result):
|
|||
d = deferred_from_coro(collect_asyncgen(result))
|
||||
d.addCallback(iterate_spider_output)
|
||||
return d
|
||||
return arg_to_iter(deferred_from_coro(result))
|
||||
elif inspect.iscoroutine(result):
|
||||
d = deferred_from_coro(result)
|
||||
d.addCallback(iterate_spider_output)
|
||||
return d
|
||||
return arg_to_iter(result)
|
||||
|
||||
|
||||
def iter_spider_classes(module):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import OpenSSL
|
||||
import OpenSSL._util as pyOpenSSLutil
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains):
|
|||
|
||||
def url_is_from_spider(url, spider):
|
||||
"""Return True if the url belongs to the given spider"""
|
||||
return url_is_from_any_domain(url,
|
||||
[spider.name] + list(getattr(spider, 'allowed_domains', [])))
|
||||
return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', [])))
|
||||
|
||||
|
||||
def url_has_any_extension(url, extensions):
|
||||
|
|
|
|||
3
setup.py
3
setup.py
|
|
@ -66,7 +66,7 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=3.5',
|
||||
python_requires='>=3.5.2',
|
||||
install_requires=[
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
|
|
@ -80,6 +80,7 @@ setup(
|
|||
'w3lib>=1.17.0',
|
||||
'zope.interface>=4.1.3',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
],
|
||||
extras_require=extras_require,
|
||||
)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue