RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

resolve conflicts

This commit is contained in:
BroodingKangaroo 2020-06-26 14:55:46 +03:00
parent dd96f94e8c 0c8d8c5e85
commit 77333666f1
79 changed files with 2027 additions and 579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.1.0
current_version = 2.2.0
commit = True
tag = True
tag_name = {new_version}

1
.gitignore vendored
View File

@ -15,6 +15,7 @@ htmlcov/
.pytest_cache/
.coverage.*
.cache/
.mypy_cache/
# Windows
Thumbs.db

8
.travis.yml
View File

@ -15,14 +15,14 @@ matrix:
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=typing
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=pinned
python: 3.5.1
dist: trusty
python: 3.5.2
- env: TOXENV=asyncio
python: 3.5.1 # We use additional code to support 3.5.3 and earlier
dist: trusty
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
- env: TOXENV=py
python: 3.5
- env: TOXENV=asyncio

2
README.rst
View File

@ -40,7 +40,7 @@ including a list of features.
Requirements
============
* Python 3.5.1+
* Python 3.5.2+
* Works on Linux, Windows, macOS, BSD
Install

9
docs/README.rst
View File

@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
need to install watchdog (``pip install watchdog``) and then use::
make watch
Alternative method using tox
----------------------------
To compile the documentation to HTML run the following command::
tox -e docs
Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.

2
docs/conf.py
View File

@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
@ -303,3 +304,4 @@ hoverxref_role_types = {
"mod": "tooltip",
"ref": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

5
docs/contributing.rst
View File

@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
removal, etc) in separate commits from functional changes. This will make pull
requests easier to review and more likely to get merged.
.. _coding-style:
Coding style
============
@ -163,7 +166,7 @@ Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 80 chars if it improves the code
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* Don't put your name in the code you contribute; git provides enough

23
docs/faq.rst
View File

@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 3.5.1+
Scrapy is supported under Python 3.5.2+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
@ -342,15 +342,15 @@ method for this purpose. For example::
from copy import deepcopy
from scrapy.item import Item
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if isinstance(item, (Item, dict)):
for _ in range(item['multiply_by']):
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
@ -371,6 +371,19 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
.. _faq-stop-response-download:
How can I cancel the download of a given response?
--------------------------------------------------
In some situations, it might be useful to stop the download of a certain response.
For instance, if you only need the first part of a large response and you would like
to save resources by avoiding the download of the whole body.
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
the :ref:`topics-stop-response-download` topic for additional information and examples.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)

2
docs/intro/install.rst
View File

@ -7,7 +7,7 @@ Installation guide
Installing Scrapy
=================
Scrapy runs on Python 3.5.1 or above under CPython (default Python
Scrapy runs on Python 3.5.2 or above under CPython (default Python
implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from

195
docs/news.rst
View File

@ -3,6 +3,201 @@
Release notes
=============
.. _release-2.2.0:
Scrapy 2.2.0 (2020-06-24)
-------------------------
Highlights:
* Python 3.5.2+ is required now
* :ref:`dataclass objects <dataclass-items>` and
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
<item-types>`
* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
* New :signal:`bytes_received` signal that allows canceling response download
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
run with a Python version lower than 3.5.2, which introduced
:class:`typing.Type` (:issue:`4615`)
Deprecations
~~~~~~~~~~~~
* :meth:`TextResponse.body_as_unicode
<scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
:attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
(:issue:`4546`, :issue:`4555`, :issue:`4579`)
* :class:`scrapy.item.BaseItem` is now deprecated, use
:class:`scrapy.item.Item` instead (:issue:`4534`)
New features
~~~~~~~~~~~~
* :ref:`dataclass objects <dataclass-items>` and
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
<item-types>`, and a new itemadapter_ library makes it easy to
write code that :ref:`supports any item type <supporting-item-types>`
(:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
* A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
:issue:`4574`)
* A new :signal:`bytes_received` signal allows monitoring response download
progress and :ref:`stopping downloads <topics-stop-response-download>`
(:issue:`4205`, :issue:`4559`)
* The dictionaries in the result list of a :ref:`media pipeline
<topics-media-pipeline>` now include a new key, ``status``, which indicates
if the file was downloaded or, if the file was not downloaded, why it was
not downloaded; see :meth:`FilesPipeline.get_media_requests
<scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
information (:issue:`2893`, :issue:`4486`)
* When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
the configured credentials do not grant the required permissions
(:issue:`4346`, :issue:`4508`)
* :ref:`Link extractors <topics-link-extractors>` are now serializable,
as long as you do not use :ref:`lambdas <lambda>` for parameters; for
example, you can now pass link extractors in :attr:`Request.cb_kwargs
<scrapy.http.Request.cb_kwargs>` or
:attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
scheduled requests <topics-jobs>` (:issue:`4554`)
* Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
from protocol 2 to protocol 4, improving serialization capabilities and
performance (:issue:`4135`, :issue:`4541`)
* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
exception if the resulting instance is ``None`` (:issue:`4528`,
:issue:`4532`)
.. _itemadapter: https://github.com/scrapy/itemadapter
Bug fixes
~~~~~~~~~
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
discards cookies defined in :attr:`Request.headers
<scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
of the ``__init__`` method of :class:`~scrapy.http.Request`
(:issue:`2400`, :issue:`3575`)
* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
``False`` and the crawl yields no items, Scrapy no longer stops feed
exports after the first URI (:issue:`4621`, :issue:`4626`)
* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
syntax <topics/coroutines>` no longer need to return an iterable, and may
instead return a :class:`~scrapy.http.Request` object, an
:ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
* The :command:`startproject` command now ensures that the generated project
folders and files have the right permissions (:issue:`4604`)
* Fix a :exc:`KeyError` exception being sometimes raised from
:class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
:issue:`4599`)
* When :setting:`FEEDS` defines multiple URIs, log messages about items being
stored now contain information from the corresponding feed, instead of
always containing information about only one of the feeds (:issue:`4619`,
:issue:`4629`)
Documentation
~~~~~~~~~~~~~
* Added a new section about :ref:`accessing cb_kwargs from errbacks
<errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
:issue:`4562`)
* Removed from :doc:`topics/coroutines` the warning about the API being
experimental (:issue:`4511`, :issue:`4513`)
* Removed references to unsupported versions of :doc:`Twisted
<twisted:index>` (:issue:`4533`)
* Updated the description of the :ref:`screenshot pipeline example
<ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
<topics/coroutines>` instead of returning a
:class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
* Removed a misleading import line from the
:func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
:issue:`4587`)
* The display-on-hover behavior of internal documentation references now also
covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
<scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
:ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
* It is again possible to download the documentation for offline reading
(:issue:`4578`, :issue:`4585`)
* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
and method signatures (:issue:`4592`, :issue:`4596`)
.. _chompjs: https://github.com/Nykakin/chompjs
Quality assurance
~~~~~~~~~~~~~~~~~
* Adjusted the code base further to our :ref:`style guidelines
<coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
:issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
:issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
:issue:`4572`)
* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
:issue:`4568`)
* Improved code sharing between the :command:`crawl` and :command:`runspider`
commands (:issue:`4548`, :issue:`4552`)
* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
(:issue:`4635`)
* You may now run the :mod:`asyncio` tests with Tox on any Python version
(:issue:`4521`)
* Updated test requirements to reflect an incompatibility with pytest 5.4 and
5.4.1 (:issue:`4588`)
* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
* Configured Travis CI to also run the tests with Python 3.5.2
(:issue:`4518`, :issue:`4615`)
* Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
(:issue:`3727`)
* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
* Made use of set literals in tests (:issue:`4573`)
* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
:issue:`4522`, :issue:`4537`)
.. _release-2.1.0:
Scrapy 2.1.0 (2020-04-24)

2
docs/topics/api.rst
View File

@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares
provided while constructing the crawler, and it is created after the
arguments given in the :meth:`crawl` method.
.. method:: crawl(\*args, \**kwargs)
.. method:: crawl(*args, **kwargs)
Starts the crawler by instantiating its spider class with the given
``args`` and ``kwargs`` arguments, while setting the execution engine in

2
docs/topics/architecture.rst
View File

@ -104,7 +104,7 @@ Spiders
-------
Spiders are custom classes written by Scrapy users to parse responses and
extract items (aka scraped items) from them or additional requests to
extract :ref:`items <topics-items>` from them or additional requests to
follow. For more information see :ref:`topics-spiders`.
.. _component-pipelines:

2
docs/topics/contracts.rst
View File

@ -78,7 +78,7 @@ override three methods:
.. module:: scrapy.contracts
.. class:: Contract(method, \*args)
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function

13
docs/topics/coroutines.rst
View File

@ -53,21 +53,28 @@ There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
item['field'] = data
adapter = ItemAdapter(item)
adapter['field'] = data
return item
def process_item(self, item, spider):
dfd = db.get_some_data(item['id'])
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
item['field'] = await db.get_some_data(item['id'])
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
return item
Coroutines may be used to call asynchronous code. This includes other

5
docs/topics/downloader-middleware.rst
View File

@ -202,6 +202,11 @@ CookiesMiddleware
sends them back on subsequent requests (from that spider), just like web
browsers do.
.. caution:: When non-UTF8 encoded byte sequences are passed to a
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`

49
docs/topics/exceptions.rst
View File

@ -14,13 +14,6 @@ Built-in Exceptions reference
Here's a list of all exceptions included in Scrapy and their usage.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
CloseSpider
-----------
@ -47,6 +40,14 @@ DontCloseSpider
This exception can be raised in a :signal:`spider_idle` signal handler to
prevent the spider from being closed.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
IgnoreRequest
-------------
@ -77,3 +78,37 @@ NotSupported
This exception is raised to indicate an unsupported feature.
StopDownload
-------------
.. versionadded:: 2.2
.. exception:: StopDownload(fail=True)
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
indicate that no further bytes should be downloaded for a response.
The ``fail`` boolean parameter controls which method will handle the resulting
response:
* If ``fail=True`` (default), the request errback is called. The response object is
available as the ``response`` attribute of the ``StopDownload`` exception,
which is in turn stored as the ``value`` attribute of the received
:class:`~twisted.python.failure.Failure` object. This means that in an errback
defined as ``def errback(self, failure)``, the response can be accessed though
``failure.value.response``.
* If ``fail=False``, the request callback is called instead.
In both cases, the response could have its body truncated: the body contains
all bytes received up until the exception is raised, including the bytes
received in the signal handler that raises the exception. Also, the response
object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
attribute.
.. note:: ``fail`` is a keyword-only parameter, i.e. raising
``StopDownload(False)`` or ``StopDownload(True)`` will raise
a :class:`TypeError`.
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
and the :ref:`topics-stop-response-download` topic for additional information and examples.

46
docs/topics/exporters.rst
View File

@ -40,6 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline:
@ -53,7 +54,8 @@ value of one of their fields::
exporter.finish_exporting()
def _exporter_for_item(self, item):
year = item['year']
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
@ -167,9 +169,10 @@ BaseItemExporter
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
:param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
:type field: :class:`~scrapy.item.Field` object or an empty dict
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
:class:`dict`.
:type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
:param name: the name of the field being serialized
:type name: str
@ -192,14 +195,17 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or None if you
want to export all fields. Defaults to None.
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Some exporters may require fields_to_export list in order to export the
data properly when spiders return dicts (not :class:`~Item` instances).
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
.. attribute:: export_empty_fields
@ -236,9 +242,9 @@ PythonItemExporter
XmlItemExporter
---------------
.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
Exports Items in XML format to the specified file object.
Exports items in XML format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -290,9 +296,9 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
Exports Items in CSV format to the given file-like object. If the
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
@ -323,9 +329,9 @@ CsvItemExporter
PickleItemExporter
------------------
.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
Exports Items in pickle format to the given file-like object.
Exports items in pickle format to the given file-like object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -343,9 +349,9 @@ PickleItemExporter
PprintItemExporter
------------------
.. class:: PprintItemExporter(file, \**kwargs)
.. class:: PprintItemExporter(file, **kwargs)
Exports Items in pretty print format to the specified file object.
Exports items in pretty print format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -363,9 +369,9 @@ PprintItemExporter
JsonItemExporter
----------------
.. class:: JsonItemExporter(file, \**kwargs)
.. class:: JsonItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing all
Exports items in JSON format to the specified file-like object, writing all
objects as a list of objects. The additional ``__init__`` method arguments are
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
@ -392,9 +398,9 @@ JsonItemExporter
JsonLinesItemExporter
---------------------
.. class:: JsonLinesItemExporter(file, \**kwargs)
.. class:: JsonLinesItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing one
Exports items in JSON format to the specified file-like object, writing one
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any

4
docs/topics/feed-exports.rst
View File

@ -299,8 +299,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
defined in dicts or :class:`~.Item` subclasses a spider is yielding.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS

66
docs/topics/item-pipeline.rst
View File

@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following
.. method:: process_item(self, item, spider)
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
(or any descendant class) object, return a
:class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
This method is called for every item pipeline component.
:param item: the item scraped
:type item: :class:`~scrapy.item.Item` object or a dict
`item` is an :ref:`item object <item-types>`, see
:ref:`supporting-item-types`.
:meth:`process_item` must either: return an :ref:`item object <item-types>`,
return a :class:`~twisted.internet.defer.Deferred` or raise a
:exc:`~scrapy.exceptions.DropItem` exception.
Dropped items are no longer processed by further pipeline components.
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
if item.get('price'):
if item.get('price_excludes_vat'):
item['price'] = item['price'] * self.vat_factor
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
@ -103,6 +108,8 @@ format::
import json
from itemadapter import ItemAdapter
class JsonWriterPipeline:
def open_spider(self, spider):
@ -112,7 +119,7 @@ format::
self.file.close()
def process_item(self, item, spider):
line = json.dumps(dict(item)) + "\n"
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@ -131,6 +138,7 @@ The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline:
@ -155,7 +163,7 @@ method and how to clean up the resources properly.::
self.client.close()
def process_item(self, item, spider):
self.db[self.collection_name].insert_one(dict(item))
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
.. _MongoDB: https://www.mongodb.com/
@ -167,18 +175,21 @@ method and how to clean up the resources properly.::
Take screenshot of item
-----------------------
This example demonstrates how to return a
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded,
it saves the screenshot to a file and adds filename to the item.
This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
the :meth:`process_item` method.
This item pipeline makes a request to a locally-running instance of Splash_ to
render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
import scrapy
import hashlib
from urllib.parse import quote
import scrapy
from itemadapter import ItemAdapter
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
@ -187,7 +198,8 @@ it saves the screenshot to a file and adds filename to the item.
SPLASH_URL = "http://localhost:8050/render.png?url={}"
async def process_item(self, item, spider):
encoded_item_url = quote(item["url"])
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
response = await spider.crawler.engine.download(request, spider)
@ -197,14 +209,14 @@ it saves the screenshot to a file and adds filename to the item.
return item
# Save screenshot to file, filename will be hash of url.
url = item["url"]
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
with open(filename, "wb") as f:
f.write(response.body)
# Store filename in item.
item["screenshot_filename"] = filename
adapter["screenshot_filename"] = filename
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
@ -217,6 +229,7 @@ already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline:
@ -225,10 +238,11 @@ returns multiples items with the same id::
self.ids_seen = set()
def process_item(self, item, spider):
if item['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %s" % item)
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item)
else:
self.ids_seen.add(item['id'])
self.ids_seen.add(adapter['id'])
return item

241
docs/topics/items.rst
View File

@ -8,29 +8,155 @@ Items
:synopsis: Item and Field classes
The main goal in scraping is to extract structured data from unstructured
sources, typically, web pages. Scrapy spiders can return the extracted data
as Python dicts. While convenient and familiar, Python dicts lack structure:
it is easy to make a typo in a field name or return inconsistent data,
especially in a larger project with many spiders.
sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
extracted data as `items`, Python objects that define key-value pairs.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide an API similar to :class:`dict` API with a convenient syntax
for declaring their available fields.
Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
item, you may use whichever type of item you want. When you write code that
receives an item, your code should :ref:`work for any item type
<supporting-item-types>`.
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _item-types:
Item Types
==========
Scrapy supports the following types of items, via the `itemadapter`_ library:
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
.. _itemadapter: https://github.com/scrapy/itemadapter
.. _dict-items:
Dictionaries
------------
As an item type, :class:`dict` is convenient and familiar.
.. _item-objects:
Item objects
------------
:class:`Item` provides a :class:`dict`-like API plus additional features that
make it the most feature-complete item type:
.. class:: Item([arg])
:class:`Item` objects replicate the standard :class:`dict` API, including
its ``__init__`` method.
:class:`Item` allows defining field names, so that:
- :class:`KeyError` is raised when using undefined field names (i.e.
prevents typos going unnoticed)
- :ref:`Item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all
of them
:class:`Item` also allows defining field metadata, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
(see :ref:`topics-leaks-trackrefs`).
:class:`Item` objects also provide the following additional API members:
.. automethod:: copy
.. automethod:: deepcopy
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
.. _dataclass-items:
Dataclass objects
-----------------
.. versionadded:: 2.2
:func:`~dataclasses.dataclass` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``dataclass`` items also allow to:
* define the type and default value of each defined field.
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
another_field: int
.. note:: Field types are not enforced at run time.
.. _attrs-items:
attr.s objects
--------------
.. versionadded:: 2.2
:func:`attr.s` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``attr.s`` items also allow to:
* define the type and default value of each defined field.
* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
another_field = attr.ib()
Working with Item objects
=========================
.. _topics-items-declaring:
Declaring Items
===============
Declaring Item subclasses
-------------------------
Items are declared using a simple class definition syntax and :class:`Field`
objects. Here is an example::
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
import scrapy
@ -48,10 +174,11 @@ objects. Here is an example::
.. _Django: https://www.djangoproject.com/
.. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
.. _topics-items-fields:
Item Fields
===========
Declaring fields
----------------
:class:`Field` objects are used to specify metadata for each field. For
example, the serializer function for the ``last_updated`` field illustrated in
@ -72,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item
do not stay assigned as class attributes. Instead, they can be accessed through
the :attr:`Item.fields` attribute.
Working with Items
==================
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
items. Please refer to the documentation for `dataclasses.field`_ and
`attr.ib`_ for additional information.
.. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
.. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
Working with Item objects
-------------------------
Here are some examples of common tasks performed with items, using the
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
notice the API is very similar to the :class:`dict` API.
Creating items
--------------
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
@ -88,7 +231,7 @@ Product(name='Desktop PC', price=1000)
Getting field values
--------------------
''''''''''''''''''''
>>> product['name']
Desktop PC
@ -128,7 +271,7 @@ False
Setting field values
--------------------
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
@ -141,7 +284,7 @@ KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
''''''''''''''''''''''''''''''
To access all populated values, just use the typical :class:`dict` API:
@ -155,7 +298,7 @@ To access all populated values, just use the typical :class:`dict` API:
.. _copying-items:
Copying items
-------------
'''''''''''''
To copy an item, you must first decide whether you want a shallow copy or a
deep copy.
@ -183,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
Other common tasks
------------------
''''''''''''''''''
Creating dicts from items:
@ -201,8 +344,8 @@ Traceback (most recent call last):
KeyError: 'Product does not support field: lala'
Extending Items
===============
Extending Item subclasses
-------------------------
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
@ -222,39 +365,25 @@ appending more values, or changing existing values, like this::
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.
Item objects
============
.. class:: Item([arg])
.. _supporting-item-types:
Return a new Item optionally initialized from the given argument.
Supporting All Item Types
=========================
Items replicate the standard :class:`dict` API, including its ``__init__``
method, and also provide the following additional API members:
In code that receives an item, such as methods of :ref:`item pipelines
<topics-item-pipeline>` or :ref:`spider middlewares
<topics-spider-middleware>`, it is a good practice to use the
:class:`~itemadapter.ItemAdapter` class and the
:func:`~itemadapter.is_item` function to write code that works for
any :ref:`supported item type <item-types>`:
.. automethod:: copy
.. autoclass:: itemadapter.ItemAdapter
.. automethod:: deepcopy
.. autofunction:: itemadapter.is_item
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Field objects
=============
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
Other classes related to Item
=============================
Other classes related to items
==============================
.. autoclass:: ItemMeta

15
docs/topics/leaks.rst
View File

@ -4,7 +4,7 @@
Debugging memory leaks
======================
In Scrapy, objects such as Requests, Responses and Items have a finite
In Scrapy, objects such as requests, responses and items have a finite
lifetime: they are created, used for a while, and finally destroyed.
From all those objects, the Request is probably the one with the longest
@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
========================================
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
memory leaks. It basically tracks the references to all live Requests,
Responses, Item and Selector objects.
memory leaks. It basically tracks the references to all live Request,
Response, Item, Spider and Selector objects.
You can enter the telnet console and inspect how many objects (of the classes
mentioned above) are currently alive using the ``prefs()`` function which is an
@ -200,11 +200,10 @@ Debugging memory leaks with muppy
``trackref`` provides a very convenient mechanism for tracking down memory
leaks, but it only keeps track of the objects that are more likely to cause
memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the muppy library.
memory leaks. However, there are other cases where the memory leaks could come
from other (more or less obscure) objects. If this is your case, and you can't
find your leaks using ``trackref``, you still have another resource: the muppy
library.
You can use muppy from `Pympler`_.

114
docs/topics/loaders.rst
View File

@ -7,13 +7,12 @@ Item Loaders
.. module:: scrapy.loader
:synopsis: Item Loader class
Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
<topics-items>`. Even though Items can be populated using their own
dictionary-like API, Item Loaders provide a much more convenient API for
populating them from a scraping process, by automating some common tasks like
parsing the raw extracted data before assigning it.
Item Loaders provide a convenient mechanism for populating scraped :ref:`items
<topics-items>`. Even though items can be populated directly, Item Loaders provide a
much more convenient API for populating them from a scraping process, by automating
some common tasks like parsing the raw extracted data before assigning it.
In other words, :ref:`Items <topics-items>` provide the *container* of
In other words, :ref:`items <topics-items>` provide the *container* of
scraped data, while Item Loaders provide the mechanism for *populating* that
container.
@ -25,10 +24,10 @@ Using Item Loaders to populate items
====================================
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
instantiate it with an :ref:`item object <topics-items>` or without one, in which
case an :ref:`item object <topics-items>` is automatically created in the
Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
specified in the :attr:`ItemLoader.default_item_class` attribute.
Then, you start collecting values into the Item Loader, typically using
:ref:`Selectors <topics-selectors>`. You can add more than one value to
@ -77,6 +76,43 @@ called which actually returns the item populated with the data
previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
.. _topics-loaders-dataclass:
Working with dataclass items
============================
By default, :ref:`dataclass items <dataclass-items>` require all fields to be
passed when created. This could be an issue when using dataclass items with
item loaders: unless a pre-populated item is passed to the loader, fields
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
Given the way that item loaders store data internally, one approach
to overcome this is to define items using the :func:`~dataclasses.field`
function, with ``list`` as the ``default_factory`` argument::
from dataclasses import dataclass, field
@dataclass
class InventoryItem:
name: str = field(default_factory=list)
price: float = field(default_factory=list)
stock: int = field(default_factory=list)
Note that in order to keep the example simple, the types do not match
completely. A more accurate but verbose definition would be::
from dataclasses import dataclass, field
from typing import List, Union
@dataclass
class InventoryItem:
name: Union[str, List[str]] = field(default_factory=list)
price: Union[float, List[float]] = field(default_factory=list)
stock: Union[int, List[int]] = field(default_factory=list)
.. _topics-loaders-processors:
Input and Output processors
@ -88,7 +124,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
:meth:`~ItemLoader.add_value` methods) and the result of the input processor is
collected and kept inside the ItemLoader. After collecting all data, the
:meth:`ItemLoader.load_item` method is called to populate and get the populated
:class:`~scrapy.item.Item` object. That's when the output processor is
:ref:`item object <topics-items>`. That's when the output processor is
called with the data previously collected (and processed using the input
processor). The result of the output processor is the final value that gets
assigned to the item.
@ -153,12 +189,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
<topics-loaders-available-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared like Items, by using a class definition syntax. Here
is an example::
Item Loaders are declared using a class definition syntax. Here is an example::
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose, Join
@ -273,11 +307,11 @@ There are several ways to modify Item Loader context values:
ItemLoader objects
==================
.. class:: ItemLoader([item, selector, response], \**kwargs)
.. class:: ItemLoader([item, selector, response], **kwargs)
Return a new Item Loader for populating the given Item. If no item is
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
Return a new Item Loader for populating the given :ref:`item object
<topics-items>`. If no item object is given, one is instantiated
automatically using the class in :attr:`default_item_class`.
When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
@ -286,7 +320,7 @@ ItemLoader objects
:param item: The item instance to populate using subsequent calls to
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
or :meth:`~ItemLoader.add_value`.
:type item: :class:`~scrapy.item.Item` object
:type item: :ref:`item object <topics-items>`
:param selector: The selector to extract data from, when using the
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
@ -303,7 +337,7 @@ ItemLoader objects
:class:`ItemLoader` instances have the following methods:
.. method:: get_value(value, \*processors, \**kwargs)
.. method:: get_value(value, *processors, **kwargs)
Process the given ``value`` by the given ``processors`` and keyword
arguments.
@ -321,7 +355,7 @@ ItemLoader objects
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
.. method:: add_value(field_name, value, *processors, **kwargs)
Process and then add the given ``value`` for the given field.
@ -343,11 +377,11 @@ ItemLoader objects
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
.. method:: replace_value(field_name, value, \*processors, \**kwargs)
.. method:: replace_value(field_name, value, *processors, **kwargs)
Similar to :meth:`add_value` but replaces the collected data with the
new value instead of adding it.
.. method:: get_xpath(xpath, \*processors, \**kwargs)
.. method:: get_xpath(xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
@ -367,7 +401,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
.. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
@ -385,12 +419,12 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
.. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`add_xpath` but replaces collected data instead of
adding it.
.. method:: get_css(css, \*processors, \**kwargs)
.. method:: get_css(css, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
@ -410,7 +444,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
.. method:: add_css(field_name, css, \*processors, \**kwargs)
.. method:: add_css(field_name, css, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
@ -428,7 +462,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_css('price', 'p#price', re='the price is (.*)')
.. method:: replace_css(field_name, css, \*processors, \**kwargs)
.. method:: replace_css(field_name, css, *processors, **kwargs)
Similar to :meth:`add_css` but replaces collected data instead of
adding it.
@ -444,17 +478,19 @@ ItemLoader objects
Create a nested loader with an xpath selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: nested_css(css)
Create a nested loader with a css selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: get_collected_values(field_name)
@ -477,7 +513,7 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
The :ref:`item object <topics-items>` being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
@ -488,8 +524,8 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
the ``__init__`` method.
An :ref:`item object <topics-items>` class or factory, used to
instantiate items when not given in the ``__init__`` method.
.. attribute:: default_input_processor
@ -678,7 +714,7 @@ Here is a list of all built-in processors:
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(\*functions, \**default_loader_context)
.. class:: Compose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions. This means that each input value of this processor is passed to
@ -706,7 +742,7 @@ Here is a list of all built-in processors:
active Loader context accessible through the :meth:`ItemLoader.context`
attribute.
.. class:: MapCompose(\*functions, \**default_loader_context)
.. class:: MapCompose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions, similar to the :class:`Compose` processor. The difference with

3
docs/topics/logging.rst
View File

@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending
.. autoclass:: scrapy.logformatter.LogFormatter
:members:
.. _topics-logging-advanced-customization:
Advanced customization
----------------------

42
docs/topics/media-pipeline.rst
View File

@ -156,7 +156,7 @@ following forms::
ftp://username:password@address:port/path
ftp://address:port/path
If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
:setting:`FTP_PASSWORD` settings respectively.
@ -201,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
Google Cloud Storage
---------------------
@ -243,20 +246,22 @@ Usage example
.. setting:: IMAGES_URLS_FIELD
.. setting:: IMAGES_RESULT_FIELD
In order to use a media pipeline first, :ref:`enable it
In order to use a media pipeline, first :ref:`enable it
<topics-media-pipeline-enabling>`.
Then, if a spider returns a dict with the URLs key (``file_urls`` or
``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
put the results under respective key (``files`` or ``images``).
Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
respectively), the pipeline will put the results under the respective field
(``files`` or ``images``).
If you prefer to use :class:`~.Item`, then define a custom item with the
necessary fields, like in this example for Images Pipeline::
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
images = scrapy.Field()
@ -445,8 +450,11 @@ See here the methods that you can override in your custom Files Pipeline:
:meth:`~get_media_requests` method and return a Request for each
file URL::
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
for file_url in item['file_urls']:
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -470,7 +478,11 @@ See here the methods that you can override in your custom Files Pipeline:
* ``checksum`` - a `MD5 hash`_ of the image contents
* ``status`` - the file status indication. It can be one of the following:
* ``status`` - the file status indication.
.. versionadded:: 2.2
It can be one of the following:
* ``downloaded`` - file was downloaded.
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
@ -509,13 +521,15 @@ See here the methods that you can override in your custom Files Pipeline:
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
item['file_paths'] = file_paths
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -589,8 +603,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy
from scrapy.pipelines.images import ImagesPipeline
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
@ -602,7 +617,8 @@ above::
image_paths = [x['path'] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
item['image_paths'] = image_paths
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
return item

83
docs/topics/request-response.rst
View File

@ -189,6 +189,10 @@ Request objects
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
In case of a failure to process the request, this dict can be accessed as
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@ -312,6 +316,31 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
----------------------------------------------
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
)
.. _topics-request-meta:
Request.meta special keys
@ -385,6 +414,51 @@ The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. _topics-stop-response-download:
Stopping the download of a Response
===================================
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
:class:`~scrapy.signals.bytes_received` signal handler will stop the
download of a given response. See the following example::
import scrapy
class StopSpider(scrapy.Spider):
name = "stop"
start_urls = ["https://docs.scrapy.org/en/latest/"]
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
return spider
def parse(self, response):
# 'last_chars' show that the full response was not downloaded
yield {"len": len(response.text), "last_chars": response.text[-40:]}
def on_bytes_received(self, data, request, spider):
raise scrapy.exceptions.StopDownload(fail=False)
which produces the following output::
2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
{'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n \n <title>Scr'}
2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
By default, resulting responses are handled by their corresponding errbacks. To
call their callback instead, like in this example, pass ``fail=False`` to the
:exc:`~scrapy.exceptions.StopDownload` exception.
.. _topics-request-response-ref-request-subclasses:
Request subclasses
@ -716,9 +790,9 @@ Response objects
.. versionadded:: 2.1.0
The IP address of the server from which the Response originated.
This attribute is currently only populated by the HTTP 1.1 download
handler, i.e. for ``http(s)`` responses. For other handlers,
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. method:: Response.copy()
@ -834,6 +908,11 @@ TextResponse objects
.. automethod:: TextResponse.follow_all
.. automethod:: TextResponse.json()
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
HtmlResponse objects
--------------------

4
docs/topics/settings.rst
View File

@ -236,8 +236,8 @@ CONCURRENT_ITEMS
Default: ``100``
Maximum number of concurrent items (per response) to process in parallel in the
Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
Maximum number of concurrent items (per response) to process in parallel in
:ref:`item pipelines <topics-item-pipeline>`.
.. setting:: CONCURRENT_REQUESTS

21
docs/topics/signals.rst
View File

@ -151,8 +151,8 @@ item_scraped
This signal supports returning deferreds from its handlers.
:param item: the item scraped
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -172,7 +172,7 @@ item_dropped
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -196,8 +196,8 @@ item_error
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
:type item: :ref:`item object <item-types>`
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@ -373,6 +373,8 @@ request_left_downloader
bytes_received
~~~~~~~~~~~~~~
.. versionadded:: 2.2
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
@ -385,14 +387,19 @@ bytes_received
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type spider: :class:`bytes` object
:type data: :class:`bytes` object
:param request: the request that generated the response
:param request: the request that generated the download
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: Handlers of this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
Response signals
----------------

13
docs/topics/spider-middleware.rst
View File

@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
objects.
:class:`~scrapy.http.Request` objects and :ref:`item object
<topics-items>`.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.http.Request`, dict
or :class:`~scrapy.item.Item` objects
:type result: an iterable of :class:`~scrapy.http.Request` objects and
:ref:`item object <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: process_spider_exception(response, exception, spider)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Request`, dict or
:class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
<topics-items>`.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_spider_exception` in the following

20
docs/topics/spiders.rst
View File

@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this:
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
Requests.
2. In the callback function, you parse the response (web page) and return either
dicts with extracted data, :class:`~scrapy.item.Item` objects,
2. In the callback function, you parse the response (web page) and return
:ref:`item objects <topics-items>`,
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
Those Requests will also contain a callback (maybe
the same) and will then be downloaded by Scrapy and then their
@ -121,7 +121,7 @@ scrapy.Spider
send log messages through it as described on
:ref:`topics-logging-from-spiders`.
.. method:: from_crawler(crawler, \*args, \**kwargs)
.. method:: from_crawler(crawler, *args, **kwargs)
This is the class method used by Scrapy to create your spiders.
@ -179,8 +179,8 @@ scrapy.Spider
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.http.Request` and/or
dicts or :class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
<topics-items>`.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback::
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
to give data more structure you can use :class:`~scrapy.item.Item` objects::
import scrapy
from myproject.items import MyItem
@ -364,7 +364,7 @@ CrawlSpider
This method is called for the start_urls responses. It allows to parse
the initial responses and must return either an
:class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
object, or an iterable containing any of them.
Crawling rules
@ -383,7 +383,7 @@ Crawling rules
object with that name will be used) to be called for each link extracted with
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
as its first argument and must return either a single instance or an iterable of
:class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
:ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
@ -531,7 +531,7 @@ XMLFeedSpider
(``itertag``). Receives the response and an
:class:`~scrapy.selector.Selector` for each node. Overriding this
method is mandatory. Otherwise, you spider won't work. This method
must return either a :class:`~scrapy.item.Item` object, a
must return an :ref:`item object <topics-items>`, a
:class:`~scrapy.http.Request` object, or an iterable containing any of
them.
@ -541,7 +541,7 @@ XMLFeedSpider
spider, and it's intended to perform any last time processing required
before returning the results to the framework core, for example setting the
item IDs. It receives a list of results and the response which originated
those results. It must return a list of results (Items or Requests).
those results. It must return a list of results (items or requests).
XMLFeedSpider example

6
pytest.ini
View File

@ -27,18 +27,16 @@ flake8-ignore =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/__init__.py E402
scrapy/selector/__init__.py F403
scrapy/spiders/__init__.py E402
scrapy/utils/http.py F403
scrapy/utils/markup.py F403
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
tests/test_webclient.py E402

2
scrapy/VERSION
View File

@ -1 +1 @@
2.1.0
2.2.0

47
scrapy/__init__.py
View File

@ -2,33 +2,11 @@
Scrapy - a web crawling and web scraping framework written for Python
"""
__all__ = ['__version__', 'version_info', 'twisted_version',
'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
# Scrapy version
import pkgutil
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
version_info = tuple(int(v) if v.isdigit() else v
for v in __version__.split('.'))
del pkgutil
# Check minimum required Python version
import sys
if sys.version_info < (3, 5):
print("Scrapy %s requires Python 3.5" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Declare top-level shortcuts
from scrapy.spiders import Spider
@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
from scrapy.selector import Selector
from scrapy.item import Item, Field
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
]
# Scrapy and Twisted versions
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 5, 2):
print("Scrapy %s requires Python 3.5.2" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del pkgutil
del sys
del warnings

11
scrapy/_monkeypatches.py
View File

@ -1,11 +0,0 @@
import copyreg
# Undo what Twisted's perspective broker adds to pickle register
# to prevent bugs like Twisted#7989 while serializing requests
import twisted.persisted.styles # NOQA
# Remove only entries with twisted serializers for non-twisted types.
for k, v in frozenset(copyreg.dispatch_table.items()):
if not str(getattr(k, '__module__', '')).startswith('twisted') \
and str(getattr(v, '__module__', '')).startswith('twisted'):
copyreg.dispatch_table.pop(k)

26
scrapy/commands/__init__.py
View File

@ -5,7 +5,7 @@ import os
from optparse import OptionGroup
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
@ -104,3 +104,27 @@ class ScrapyCommand:
Entry point for running commands
"""
raise NotImplementedError
class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
self.settings.set('FEEDS', feeds, priority='cmdline')

24
scrapy/commands/crawl.py
View File

@ -1,9 +1,8 @@
from scrapy.commands import ScrapyCommand
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = True
@ -13,25 +12,6 @@ class Command(ScrapyCommand):
def short_desc(self):
return "Run a spider"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
self.settings.set('FEEDS', feeds, priority='cmdline')
def run(self, args, opts):
if len(args) < 1:
raise UsageError()

6
scrapy/commands/parse.py
View File

@ -1,11 +1,11 @@
import json
import logging
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.item import _BaseItem
from scrapy.utils import display
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@ -81,7 +81,7 @@ class Command(ScrapyCommand):
items = self.items.get(lvl, [])
print("# Scraped Items ", "-" * 60)
display.pprint([dict(x) for x in items], colorize=colour)
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl=None, colour=True):
if lvl is None:
@ -117,7 +117,7 @@ class Command(ScrapyCommand):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (_BaseItem, dict)):
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)

24
scrapy/commands/runspider.py
View File

@ -3,9 +3,8 @@ import os
from importlib import import_module
from scrapy.utils.spider import iter_spider_classes
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath):
@ -24,7 +23,7 @@ def _import_file(filepath):
return module
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
@ -38,25 +37,6 @@ class Command(ScrapyCommand):
def long_desc(self):
return "Run the spider defined in the given file"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
self.settings.set('FEEDS', feeds, priority='cmdline')
def run(self, args, opts):
if len(args) != 1:
raise UsageError()

24
scrapy/commands/startproject.py
View File

@ -1,5 +1,6 @@
import re
import os
import stat
import string
from importlib import import_module
from os.path import join, exists, abspath
@ -78,6 +79,29 @@ class Command(ScrapyCommand):
else:
copy2(srcname, dstname)
copystat(src, dst)
self._set_rw_permissions(dst)
def _set_rw_permissions(self, path):
"""
Sets permissions of a directory tree to +rw and +rwx for folders.
This is necessary if the start template files come without write
permissions.
"""
mode_rw = (stat.S_IRUSR
| stat.S_IWUSR
| stat.S_IRGRP
| stat.S_IROTH)
mode_x = (stat.S_IXUSR
| stat.S_IXGRP
| stat.S_IXOTH)
os.chmod(path, mode_rw | mode_x)
for root, dirs, files in os.walk(path):
for dir in dirs:
os.chmod(join(root, dir), mode_rw | mode_x)
for file in files:
os.chmod(join(root, file), mode_rw)
def run(self, args, opts):
if len(args) not in (1, 2):

28
scrapy/contracts/default.py
View File

@ -1,10 +1,10 @@
import json
from scrapy.item import _BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
from itemadapter import is_item, ItemAdapter
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
from scrapy.http import Request
# contracts
@ -48,11 +48,11 @@ class ReturnsContract(Contract):
"""
name = 'returns'
objects = {
'request': Request,
'requests': Request,
'item': (_BaseItem, dict),
'items': (_BaseItem, dict),
object_type_verifiers = {
'request': lambda x: isinstance(x, Request),
'requests': lambda x: isinstance(x, Request),
'item': is_item,
'items': is_item,
}
def __init__(self, *args, **kwargs):
@ -64,7 +64,7 @@ class ReturnsContract(Contract):
% len(self.args)
)
self.obj_name = self.args[0] or None
self.obj_type = self.objects[self.obj_name]
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
try:
self.min_bound = int(self.args[1])
@ -79,7 +79,7 @@ class ReturnsContract(Contract):
def post_process(self, output):
occurrences = 0
for x in output:
if isinstance(x, self.obj_type):
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
@ -103,8 +103,8 @@ class ScrapesContract(Contract):
def post_process(self, output):
for x in output:
if isinstance(x, (_BaseItem, dict)):
missing = [arg for arg in self.args if arg not in x]
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
raise ContractFail(
"Missing fields: %s" % ", ".join(missing))
missing_str = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str)

53
scrapy/core/downloader/handlers/http11.py
View File

@ -12,6 +12,7 @@ from urllib.parse import urldefrag
from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http_headers import Headers as TxHeaders
@ -21,7 +22,7 @@ from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.misc import create_instance, load_object
@ -431,7 +432,7 @@ class ScrapyAgent:
def _cb_bodydone(self, result, request, url):
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
return respcls(
response = respcls(
url=url,
status=int(result["txresponse"].code),
headers=headers,
@ -440,6 +441,10 @@ class ScrapyAgent:
certificate=result["certificate"],
ip_address=result["ip_address"],
)
if result.get("failure"):
result["failure"].value.response = response
return result["failure"]
return response
@implementer(IBodyProducer)
@ -477,6 +482,16 @@ class _ResponseReader(protocol.Protocol):
self._ip_address = None
self._crawler = crawler
def _finish_response(self, flags=None, failure=None):
self._finished.callback({
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
})
def connectionMade(self):
if self._certificate is None:
with suppress(AttributeError):
@ -493,12 +508,19 @@ class _ResponseReader(protocol.Protocol):
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
self._crawler.signals.send_catch_log(
bytes_received_result = self._crawler.signals.send_catch_log(
signal=signals.bytes_received,
data=bodyBytes,
request=self._request,
spider=self._crawler.spider,
)
for handler, result in bytes_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
self.transport._producer.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
@ -521,36 +543,17 @@ class _ResponseReader(protocol.Protocol):
if self._finished.called:
return
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": None,
"certificate": self._certificate,
"ip_address": self._ip_address,
})
self._finish_response()
return
if reason.check(PotentialDataLoss):
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["partial"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
self._finish_response(flags=["partial"])
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["dataloss"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
self._finish_response(flags=["dataloss"])
return
elif not self._fail_on_dataloss_warned:

27
scrapy/core/scraper.py
View File

@ -4,18 +4,18 @@ extracts information from them"""
import logging
from collections import deque
from twisted.python.failure import Failure
from itemadapter import is_item
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy import signals
from scrapy.http import Request, Response
from scrapy.item import _BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
logger = logging.getLogger(__name__)
@ -191,7 +191,7 @@ class Scraper:
"""
if isinstance(output, Request):
self.crawler.engine.crawl(request=output, spider=spider)
elif isinstance(output, (_BaseItem, dict)):
elif is_item(output):
self.slot.itemproc_size += 1
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, response, spider)
@ -200,10 +200,11 @@ class Scraper:
pass
else:
typename = type(output).__name__
logger.error('Spider must return Request, BaseItem, dict or None, '
'got %(typename)r in %(request)s',
{'request': request, 'typename': typename},
extra={'spider': spider})
logger.error(
'Spider must return request, item, or None, got %(typename)r in %(request)s',
{'request': request, 'typename': typename},
extra={'spider': spider},
)
def _log_download_errors(self, spider_failure, download_failure, request, spider):
"""Log and silence errors that come from the engine (typically download

78
scrapy/downloadermiddlewares/cookies.py
View File

@ -29,8 +29,7 @@ class CookiesMiddleware:
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
cookies = self._get_request_cookies(jar, request)
for cookie in cookies:
for cookie in self._get_request_cookies(jar, request):
jar.set_cookie_if_ok(cookie, request)
# set Cookie header
@ -68,28 +67,65 @@ class CookiesMiddleware:
msg = "Received cookies from: {}\n{}".format(response, cookies)
logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie):
# build cookie string
cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
if cookie.get('path', None):
cookie_str += '; Path=%s' % cookie['path']
if cookie.get('domain', None):
cookie_str += '; Domain=%s' % cookie['domain']
def _format_cookie(self, cookie, request):
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
if not cookie.get(key):
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
return
continue
if isinstance(cookie[key], str):
decoded[key] = cookie[key]
else:
try:
decoded[key] = cookie[key].decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
for key, value in decoded.items(): # path, domain
cookie_str += "; {}={}".format(key.capitalize(), value)
return cookie_str
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
cookie_list = [
{'name': k, 'value': v}
for k, v in request.cookies.items()
]
else:
cookie_list = request.cookies
"""
Extract cookies from a Request. Values from the `Request.cookies` attribute
take precedence over values from the `Cookie` request header.
"""
def get_cookies_from_header(jar, request):
cookie_header = request.headers.get("Cookie")
if not cookie_header:
return []
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
cookie_list_unicode = []
for cookie_bytes in cookie_gen_bytes:
try:
cookie_unicode = cookie_bytes.decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie_bytes)
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
cookie_list_unicode.append(cookie_unicode)
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
return jar.make_cookies(response, request)
cookies = [self._format_cookie(x) for x in cookie_list]
headers = {'Set-Cookie': cookies}
response = Response(request.url, headers=headers)
def get_cookies_from_attribute(jar, request):
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
response = Response(request.url, headers={"Set-Cookie": formatted})
return jar.make_cookies(response, request)
return jar.make_cookies(response, request)
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)

13
scrapy/exceptions.py
View File

@ -41,6 +41,18 @@ class CloseSpider(Exception):
self.reason = reason
class StopDownload(Exception):
"""
Stop the download of the body for a given response.
The 'fail' boolean parameter indicates whether or not the resulting partial response
should be handled by the request errback. Note that 'fail' is a keyword-only argument.
"""
def __init__(self, *, fail=True):
super().__init__()
self.fail = fail
# Items
@ -59,6 +71,7 @@ class NotSupported(Exception):
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)

38
scrapy/exporters.py
View File

@ -4,16 +4,18 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
import pprint
import marshal
import warnings
import pickle
import pprint
import warnings
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import _BaseItem
from itemadapter import is_item, ItemAdapter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import _BaseItem
from scrapy.utils.python import is_listlike, to_bytes, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@ -56,11 +58,14 @@ class BaseItemExporter:
"""Return the fields to export as an iterable of tuples
(name, serialized_value)
"""
item = ItemAdapter(item)
if include_empty is None:
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
field_iter = item.fields.keys()
if include_empty:
field_iter = item.field_names()
else:
field_iter = item.keys()
else:
@ -71,8 +76,8 @@ class BaseItemExporter:
for field_name in field_iter:
if field_name in item:
field = {} if isinstance(item, dict) else item.fields[field_name]
value = self.serialize_field(field, field_name, item[field_name])
field_meta = item.get_field_meta(field_name)
value = self.serialize_field(field_meta, field_name, item[field_name])
else:
value = default_value
@ -297,6 +302,7 @@ class PythonItemExporter(BaseItemExporter):
.. _msgpack: https://pypi.org/project/msgpack/
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
super(PythonItemExporter, self)._configure(options, dont_fail)
@ -314,22 +320,22 @@ class PythonItemExporter(BaseItemExporter):
def _serialize_value(self, value):
if isinstance(value, _BaseItem):
return self.export_item(value)
if isinstance(value, dict):
return dict(self._serialize_dict(value))
if is_listlike(value):
elif is_item(value):
return dict(self._serialize_item(value))
elif is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
for key, val in value.items():
def _serialize_item(self, item):
for key, value in ItemAdapter(item).items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)
yield key, self._serialize_value(value)
def export_item(self, item):
result = dict(self._get_serialized_fields(item))
if self.binary:
result = dict(self._serialize_dict(result))
result = dict(self._serialize_item(result))
return result

19
scrapy/extensions/feedexport.py
View File

@ -281,11 +281,20 @@ class FeedExporter:
'itemcount': slot.itemcount,
'uri': slot.uri}
d = defer.maybeDeferred(slot.storage.store, slot.file)
d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
extra={'spider': spider}))
d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
)
return d
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):

14
scrapy/http/response/text.py
View File

@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
import json
import warnings
from contextlib import suppress
from typing import Generator
@ -21,10 +22,13 @@ from scrapy.http.response import Response
from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
_NONE = object()
class TextResponse(Response):
_DEFAULT_ENCODING = 'ascii'
_cached_decoded_json = _NONE
def __init__(self, *args, **kwargs):
self._encoding = kwargs.pop('encoding', None)
@ -68,6 +72,16 @@ class TextResponse(Response):
ScrapyDeprecationWarning, stacklevel=2)
return self.text
def json(self):
"""
.. versionadded:: 2.2
Deserialize a JSON document to a Python object.
"""
if self._cached_decoded_json is _NONE:
self._cached_decoded_json = json.loads(self.text)
return self._cached_decoded_json
@property
def text(self):
""" Body as unicode """

2
scrapy/item.py
View File

@ -36,7 +36,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
"""
def __new__(cls, *args, **kwargs):
if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(BaseItem, cls).__new__(cls, *args, **kwargs)

2
scrapy/linkextractors/__init__.py
View File

@ -133,4 +133,4 @@ class FilteringLinkExtractor:
# Top-level imports
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor

18
scrapy/loader/__init__.py
View File

@ -6,6 +6,8 @@ See documentation in docs/topics/loaders.rst
from collections import defaultdict
from contextlib import suppress
from itemadapter import ItemAdapter
from scrapy.item import Item
from scrapy.loader.common import wrap_loader_context
from scrapy.loader.processors import Identity
@ -44,7 +46,7 @@ class ItemLoader:
self._local_item = context['item'] = item
self._local_values = defaultdict(list)
# values from initial item
for field_name, value in item.items():
for field_name, value in ItemAdapter(item).items():
self._values[field_name] += arg_to_iter(value)
@property
@ -127,13 +129,12 @@ class ItemLoader:
return value
def load_item(self):
item = self.item
adapter = ItemAdapter(self.item)
for field_name in tuple(self._values):
value = self.get_output_value(field_name)
if value is not None:
item[field_name] = value
return item
adapter[field_name] = value
return adapter.item
def get_output_value(self, field_name):
proc = self.get_output_processor(field_name)
@ -174,11 +175,8 @@ class ItemLoader:
value, type(e).__name__, str(e)))
def _get_item_field_attr(self, field_name, key, default=None):
if isinstance(self.item, Item):
value = self.item.fields[field_name].get(key, default)
else:
value = default
return value
field_meta = ItemAdapter(self.item).get_field_meta(field_name)
return field_meta.get(key, default)
def _check_selector_method(self):
if self.selector is None:

23
scrapy/pipelines/files.py
View File

@ -10,24 +10,26 @@ import mimetypes
import os
import time
from collections import defaultdict
from email.utils import parsedate_tz, mktime_tz
from contextlib import suppress
from email.utils import mktime_tz, parsedate_tz
from ftplib import FTP
from io import BytesIO
from urllib.parse import urlparse
from itemadapter import ItemAdapter
from twisted.internet import defer, threads
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.misc import md5sum
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import md5sum
from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
logger = logging.getLogger(__name__)
@ -517,7 +519,8 @@ class FilesPipeline(MediaPipeline):
# Overridable Interface
def get_media_requests(self, item, info):
return [Request(x) for x in item.get(self.files_urls_field, [])]
urls = ItemAdapter(item).get(self.files_urls_field, [])
return [Request(u) for u in urls]
def file_downloaded(self, response, request, info):
path = self.file_path(request, response=response, info=info)
@ -528,8 +531,8 @@ class FilesPipeline(MediaPipeline):
return checksum
def item_completed(self, results, item, info):
if isinstance(item, dict) or self.files_result_field in item.fields:
item[self.files_result_field] = [x for ok, x in results if ok]
with suppress(KeyError):
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):

19
scrapy/pipelines/images.py
View File

@ -5,17 +5,19 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
from contextlib import suppress
from io import BytesIO
from itemadapter import ItemAdapter
from PIL import Image
from scrapy.exceptions import DropItem
from scrapy.http import Request
from scrapy.pipelines.files import FileException, FilesPipeline
# TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.misc import md5sum
from scrapy.utils.python import to_bytes
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.exceptions import DropItem
# TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.pipelines.files import FileException, FilesPipeline
class NoimagesDrop(DropItem):
@ -157,11 +159,12 @@ class ImagesPipeline(FilesPipeline):
return image, buf
def get_media_requests(self, item, info):
return [Request(x) for x in item.get(self.images_urls_field, [])]
urls = ItemAdapter(item).get(self.images_urls_field, [])
return [Request(u) for u in urls]
def item_completed(self, results, item, info):
if isinstance(item, dict) or self.images_result_field in item.fields:
item[self.images_result_field] = [x for ok, x in results if ok]
with suppress(KeyError):
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
return item
def file_path(self, request, response=None, info=None):

4
scrapy/selector/__init__.py
View File

@ -1,4 +1,6 @@
"""
Selectors
"""
from scrapy.selector.unified import * # noqa: F401
# top-level imports
from scrapy.selector.unified import Selector, SelectorList

11
scrapy/shell.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
import os
import signal
from itemadapter import is_item
from twisted.internet import threads, defer
from twisted.python import threadable
from w3lib.url import any_to_uri
@ -13,20 +14,18 @@ from w3lib.url import any_to_uri
from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.item import _BaseItem
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.console import start_python_console
from scrapy.utils.conf import get_config
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
from scrapy.utils.conf import get_config
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS
class Shell:
relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings)
relevant_classes = (Crawler, Spider, Request, Response, Settings)
def __init__(self, crawler, update_vars=None, code=None):
self.crawler = crawler
@ -154,7 +153,7 @@ class Shell:
return "\n".join("[s] %s" % line for line in b)
def _is_relevant(self, value):
return isinstance(value, self.relevant_classes)
return isinstance(value, self.relevant_classes) or is_item(value)
def inspect_response(response, spider):

6
scrapy/spiders/__init__.py
View File

@ -110,6 +110,6 @@ class Spider(object_ref):
# Top-level imports
from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401
from scrapy.spiders.sitemap import SitemapSpider # noqa: F401
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
from scrapy.spiders.sitemap import SitemapSpider

2
scrapy/spiders/feed.py
View File

@ -31,7 +31,7 @@ class XMLFeedSpider(Spider):
processing required before returning the results to the framework core,
for example setting the item GUIDs. It receives a list of results and
the response which originated that results. It must return a list of
results (Items or Requests).
results (items or requests).
"""
return results

8
scrapy/templates/project/module/middlewares.py.tmpl
View File

@ -5,6 +5,9 @@
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
class ${ProjectName}SpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
@ -29,7 +32,7 @@ class ${ProjectName}SpiderMiddleware:
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
@ -37,8 +40,7 @@ class ${ProjectName}SpiderMiddleware:
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request, dict
# or Item objects.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):

4
scrapy/templates/project/module/pipelines.py.tmpl
View File

@ -4,6 +4,10 @@
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class ${ProjectName}Pipeline:
def process_item(self, item, spider):
return item

4
scrapy/utils/datatypes.py
View File

@ -105,8 +105,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
def __getitem__(self, key):
try:
return super(LocalWeakReferencedCache, self).__getitem__(key)
except TypeError:
return None # key is not weak-referenceable, it's not cached
except (TypeError, KeyError):
return None # key is either not weak-referenceable or not cached
class SequenceExclude:

5
scrapy/utils/misc.py
View File

@ -138,8 +138,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
extension has not been implemented correctly).
.. versionchanged:: 2.2
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
extension has not been implemented correctly).
"""
if settings is None:
if crawler is None:

4
scrapy/utils/python.py
View File

@ -334,10 +334,10 @@ class MutableChain:
"""
def __init__(self, *args):
self.data = chain(*args)
self.data = chain.from_iterable(args)
def extend(self, *iterables):
self.data = chain(self.data, *iterables)
self.data = chain(self.data, chain.from_iterable(iterables))
def __iter__(self):
return self

6
scrapy/utils/serialize.py
View File

@ -2,10 +2,10 @@ import json
import datetime
import decimal
from itemadapter import is_item, ItemAdapter
from twisted.internet import defer
from scrapy.http import Request, Response
from scrapy.item import _BaseItem
class ScrapyJSONEncoder(json.JSONEncoder):
@ -26,8 +26,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
return str(o)
elif isinstance(o, defer.Deferred):
return str(o)
elif isinstance(o, _BaseItem):
return dict(o)
elif is_item(o):
return ItemAdapter(o).asdict()
elif isinstance(o, Request):
return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
elif isinstance(o, Response):

7
scrapy/utils/signal.py
View File

@ -5,13 +5,14 @@ import logging
from twisted.internet.defer import DeferredList, Deferred
from twisted.python.failure import Failure
from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
getAllReceivers, disconnect
from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
from pydispatch.robustapply import robustApply
from scrapy.exceptions import StopDownload
from scrapy.utils.defer import maybeDeferred_coro
from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
@ -23,7 +24,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
Failures instead of exceptions.
"""
dont_log = named.pop('dont_log', _IgnoredException)
dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
spider = named.get('spider', None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):

8
scrapy/utils/spider.py
View File

@ -1,5 +1,5 @@
import logging
import inspect
import logging
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro
@ -18,7 +18,11 @@ def iterate_spider_output(result):
d = deferred_from_coro(collect_asyncgen(result))
d.addCallback(iterate_spider_output)
return d
return arg_to_iter(deferred_from_coro(result))
elif inspect.iscoroutine(result):
d = deferred_from_coro(result)
d.addCallback(iterate_spider_output)
return d
return arg_to_iter(result)
def iter_spider_classes(module):

171
setup.cfg
View File

@ -3,3 +3,174 @@ doc_files = docs AUTHORS INSTALL LICENSE README.rst
[bdist_wheel]
universal=1
[mypy]
ignore_missing_imports = true
follow_imports = skip
# FIXME: remove the following sections once the issues are solved
[mypy-scrapy]
ignore_errors = True
[mypy-scrapy._monkeypatches]
ignore_errors = True
[mypy-scrapy.commands]
ignore_errors = True
[mypy-scrapy.commands.bench]
ignore_errors = True
[mypy-scrapy.commands.parse]
ignore_errors = True
[mypy-scrapy.downloadermiddlewares.httpproxy]
ignore_errors = True
[mypy-scrapy.contracts]
ignore_errors = True
[mypy-scrapy.core.spidermw]
ignore_errors = True
[mypy-scrapy.interfaces]
ignore_errors = True
[mypy-scrapy.item]
ignore_errors = True
[mypy-scrapy.http.cookies]
ignore_errors = True
[mypy-scrapy.mail]
ignore_errors = True
[mypy-scrapy.pipelines.images]
ignore_errors = True
[mypy-scrapy.settings.default_settings]
ignore_errors = True
[mypy-scrapy.spidermiddlewares.referer]
ignore_errors = True
[mypy-scrapy.utils.httpobj]
ignore_errors = True
[mypy-scrapy.utils.request]
ignore_errors = True
[mypy-scrapy.utils.response]
ignore_errors = True
[mypy-scrapy.utils.spider]
ignore_errors = True
[mypy-scrapy.utils.trackref]
ignore_errors = True
[mypy-tests.mocks.dummydbm]
ignore_errors = True
[mypy-tests.spiders]
ignore_errors = True
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
ignore_errors = True
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
ignore_errors = True
[mypy-tests.test_command_fetch]
ignore_errors = True
[mypy-tests.test_command_parse]
ignore_errors = True
[mypy-tests.test_command_shell]
ignore_errors = True
[mypy-tests.test_command_version]
ignore_errors = True
[mypy-tests.test_contracts]
ignore_errors = True
[mypy-tests.test_crawler]
ignore_errors = True
[mypy-tests.test_downloader_handlers]
ignore_errors = True
[mypy-tests.test_engine]
ignore_errors = True
[mypy-tests.test_exporters]
ignore_errors = True
[mypy-tests.test_http_request]
ignore_errors = True
[mypy-tests.test_linkextractors]
ignore_errors = True
[mypy-tests.test_loader]
ignore_errors = True
[mypy-tests.test_pipeline_crawl]
ignore_errors = True
[mypy-tests.test_pipeline_files]
ignore_errors = True
[mypy-tests.test_pipeline_images]
ignore_errors = True
[mypy-tests.test_pipelines]
ignore_errors = True
[mypy-tests.test_request_cb_kwargs]
ignore_errors = True
[mypy-tests.test_request_left]
ignore_errors = True
[mypy-tests.test_scheduler]
ignore_errors = True
[mypy-tests.test_signals]
ignore_errors = True
[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
ignore_errors = True
[mypy-tests.test_spiderloader.test_spiders.spider1]
ignore_errors = True
[mypy-tests.test_spiderloader.test_spiders.spider2]
ignore_errors = True
[mypy-tests.test_spiderloader.test_spiders.spider3]
ignore_errors = True
[mypy-tests.test_spidermiddleware_httperror]
ignore_errors = True
[mypy-tests.test_spidermiddleware_output_chain]
ignore_errors = True
[mypy-tests.test_spidermiddleware_referer]
ignore_errors = True
[mypy-tests.test_utils_reqser]
ignore_errors = True
[mypy-tests.test_utils_serialize]
ignore_errors = True
[mypy-tests.test_utils_spider]
ignore_errors = True
[mypy-tests.test_utils_url]
ignore_errors = True

3
setup.py
View File

@ -66,7 +66,7 @@ setup(
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
python_requires='>=3.5',
python_requires='>=3.5.2',
install_requires=[
'Twisted>=17.9.0',
'cryptography>=2.0',
@ -80,6 +80,7 @@ setup(
'w3lib>=1.17.0',
'zope.interface>=4.1.3',
'protego>=0.1.15',
'itemadapter>=0.1.0',
],
extras_require=extras_require,
)

5
tests/requirements-py3.txt
View File

@ -1,8 +1,11 @@
# Tests requirements
attrs
dataclasses; python_version == '3.6'
jmespath
mitmproxy; python_version >= '3.6'
mitmproxy<4.0.0; python_version < '3.6'
pytest < 5.4
# https://github.com/pytest-dev/pytest-twisted/issues/93
pytest != 5.4, != 5.4.1
pytest-cov
pytest-twisted >= 1.11
pytest-xdist

46
tests/spiders.py
View File

@ -7,6 +7,8 @@ from urllib.parse import urlencode
from twisted.internet import defer
from scrapy import signals
from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
@ -117,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
return [{'id': 1}, {'id': 2}]
class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
name = "asyncdef_asyncio_return_single_element"
async def parse(self, response):
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d" % status)
return {"foo": 42}
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = 'asyncdef_asyncio_reqs_return'
@ -267,3 +280,36 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
def errback(self, failure):
self.logger.info('[errback] status %i', failure.value.response.status)
class BytesReceivedCallbackSpider(MetaSpider):
full_response_length = 2**18
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.bytes_received, signals.bytes_received)
return spider
def start_requests(self):
body = b"a" * self.full_response_length
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=body, errback=self.errback)
def parse(self, response):
self.meta["response"] = response
def errback(self, failure):
self.meta["failure"] = failure
def bytes_received(self, data, request, spider):
self.meta["bytes_received"] = data
raise StopDownload(fail=False)
class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
def bytes_received(self, data, request, spider):
self.meta["bytes_received"] = data
raise StopDownload(fail=True)

62
tests/test_crawl.py
View File

@ -9,17 +9,32 @@ from pytest import mark
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from scrapy import signals
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider,
AsyncDefAsyncioReqsReturnSpider)
from tests.spiders import (
AsyncDefAsyncioReqsReturnSpider,
AsyncDefAsyncioReturnSingleElementSpider,
AsyncDefAsyncioReturnSpider,
AsyncDefAsyncioSpider,
AsyncDefSpider,
BrokenStartRequestsSpider,
BytesReceivedCallbackSpider,
BytesReceivedErrbackSpider,
CrawlSpiderWithErrback,
DelaySpider,
DuplicateStartRequestsSpider,
FollowAllSpider,
SimpleSpider,
SingleRequestSpider,
)
class CrawlTestCase(TestCase):
@ -350,6 +365,21 @@ with multiples lines
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_items_single_element(self):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
self.assertIn({"foo": 42}, items)
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@mark.only_asyncio()
@defer.inlineCallbacks
@ -457,3 +487,27 @@ with multiples lines
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsInstance(ip_address, IPv4Address)
self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
@defer.inlineCallbacks
def test_stop_download_callback(self):
crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))
self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
@defer.inlineCallbacks
def test_stop_download_errback(self):
crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
self.assertEqual(
crawler.spider.meta["failure"].value.response.body,
crawler.spider.meta.get("bytes_received"))
self.assertLess(
len(crawler.spider.meta["failure"].value.response.body),
crawler.spider.full_response_length)

145
tests/test_downloadermiddleware_cookies.py
View File

@ -1,20 +1,21 @@
import re
import logging
from unittest import TestCase
from testfixtures import LogCapture
from unittest import TestCase
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
from scrapy.exceptions import NotConfigured
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
class CookiesMiddlewareTest(TestCase):
def assertCookieValEqual(self, first, second, msg=None):
def split_cookies(cookies):
return sorted(re.split(r";\s*", cookies.decode("latin1")))
return sorted([s.strip() for s in to_bytes(cookies).split(b";")])
return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
def setUp(self):
@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase):
def test_setting_enabled_cookies_debug(self):
crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as log:
with LogCapture(
'scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG,
) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
mw.process_response(req, res, crawler.spider)
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase):
def test_setting_disabled_cookies_debug(self):
crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
mw = CookiesMiddleware.from_crawler(crawler)
with LogCapture('scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG) as log:
with LogCapture(
'scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.DEBUG,
) as log:
req = Request('http://scrapytest.org/')
res = Response('http://scrapytest.org/',
headers={'Set-Cookie': 'C1=value1; path=/'})
res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
mw.process_response(req, res, crawler.spider)
req2 = Request('http://scrapytest.org/sub1/')
mw.process_request(req2, crawler.spider)
@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase):
assert self.mw.process_request(req, self.spider) is None
assert 'Cookie' not in req.headers
headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
'Other': b'ignore\xa3me'}
headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'}
res = Response('http://scrapytest.org/', headers=headers)
assert self.mw.process_response(req, res, self.spider) is res
@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase):
assert 'Cookie' not in req.headers
# check that returned cookies are not merged back to jar
res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'})
res = Response(
'http://scrapytest.org/dontmerge',
headers={'Set-Cookie': 'dont=mergeme; path=/'},
)
assert self.mw.process_response(req, res, self.spider) is res
# check that cookies are merged back
@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase):
self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
def test_cookiejar_key(self):
req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
req = Request(
'http://scrapytest.org/',
cookies={'galleta': 'salada'},
meta={'cookiejar': "store1"},
)
assert self.mw.process_request(req, self.spider) is None
self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase):
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
req3 = Request(
'http://scrapytest.org/',
cookies={'galleta': 'dulce'},
meta={'cookiejar': "store2"},
)
assert self.mw.process_request(req3, self.spider) is None
self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase):
assert self.mw.process_request(request, self.spider) is None
self.assertIn('Cookie', request.headers)
self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
def test_keep_cookie_from_default_request_headers_middleware(self):
DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
# overwrite with values from 'cookies' request argument
req1 = Request('http://example.org', cookies={'default': 'something'})
assert mw_default_headers.process_request(req1, self.spider) is None
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty')
# keep both
req2 = Request('http://example.com', cookies={'a': 'b'})
assert mw_default_headers.process_request(req2, self.spider) is None
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
def test_keep_cookie_header(self):
# keep only cookies from 'Cookie' request header
req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d')
# keep cookies from both 'Cookie' request header and 'cookies' keyword
req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f')
# overwrite values from 'Cookie' request header with 'cookies' keyword
req3 = Request(
'http://scrapytest.org',
headers={'Cookie': 'a=b; c=d'},
cookies={'a': 'new', 'e': 'f'},
)
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f')
def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
# 3) Unicode string
req3 = Request('http://example.org', cookies={'a': u'á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
# 3) Unicode string
req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
def test_invalid_cookies(self):
"""
Invalid cookies are logged as warnings and discarded
"""
with LogCapture(
'scrapy.downloadermiddlewares.cookies',
propagate=False,
level=logging.INFO,
) as lc:
cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}]
req1 = Request('http://example.org/1', cookies=cookies1)
assert self.mw.process_request(req1, self.spider) is None
cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
req2 = Request('http://example.org/2', cookies=cookies2)
assert self.mw.process_request(req2, self.spider) is None
lc.check(
("scrapy.downloadermiddlewares.cookies",
"WARNING",
"Invalid cookie found in request <GET http://example.org/1>:"
" {'value': 'bar'} ('name' is missing)"),
("scrapy.downloadermiddlewares.cookies",
"WARNING",
"Invalid cookie found in request <GET http://example.org/2>:"
" {'name': 'foo'} ('value' is missing)"),
)
self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')

97
tests/test_engine.py
View File

@ -16,13 +16,17 @@ import sys
from collections import defaultdict
from urllib.parse import urlparse
from twisted.internet import reactor, defer
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from testfixtures import LogCapture
from twisted.internet import defer, reactor
from twisted.trial import unittest
from twisted.web import server, static, util
from pydispatch import dispatcher
from scrapy import signals
from scrapy.core.engine import ExecutionEngine
from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.linkextractors import LinkExtractor
@ -30,7 +34,7 @@ from scrapy.spiders import Spider
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
from tests import tests_datadir, get_testdata
from tests import get_testdata, tests_datadir
class TestItem(Item):
@ -39,6 +43,13 @@ class TestItem(Item):
price = Field()
@attr.s
class AttrsItem:
name = attr.ib(default="")
url = attr.ib(default="")
price = attr.ib(default=0)
class TestSpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
@ -77,6 +88,27 @@ class DictItemsSpider(TestSpider):
item_cls = dict
class AttrsItemsSpider(TestSpider):
item_class = AttrsItem
try:
from dataclasses import make_dataclass
except ImportError:
DataClassItemsSpider = None
else:
TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
class DataClassItemsSpider(DictItemsSpider):
def parse_item(self, response):
item = super().parse_item(response)
return TestDataClass(
name=item.get('name'),
url=item.get('url'),
price=item.get('price'),
)
class ItemZeroDivisionErrorSpider(TestSpider):
custom_settings = {
"ITEM_PIPELINES": {
@ -90,7 +122,7 @@ def start_test_site(debug=False):
r = static.File(root_dir)
r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**14)]
numbers = [str(x).encode("utf8") for x in range(2**18)]
r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain"))
port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
@ -188,11 +220,24 @@ class CrawlerRun:
self.signals_caught[sig] = signalargs
class StopDownloadCrawlerRun(CrawlerRun):
"""
Make sure raising the StopDownload exception stops the download of the response body
"""
def bytes_received(self, data, request, spider):
super().bytes_received(data, request, spider)
raise StopDownload(fail=False)
class EngineTest(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler(self):
for spider in TestSpider, DictItemsSpider:
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
if spider is None:
continue
self.run = CrawlerRun(spider)
yield self.run.run()
self._assert_visited_urls()
@ -269,6 +314,7 @@ class EngineTest(unittest.TestCase):
def _assert_scraped_items(self):
self.assertEqual(2, len(self.run.itemresp))
for item, response in self.run.itemresp:
item = ItemAdapter(item)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
self.assertEqual('Item 1 name', item['name'])
@ -316,7 +362,7 @@ class EngineTest(unittest.TestCase):
# signal was fired multiple times
self.assertTrue(len(data) > 1)
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**14)]
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.assertEqual(joined_data, b"".join(numbers))
def _assert_signals_caught(self):
@ -357,6 +403,45 @@ class EngineTest(unittest.TestCase):
self.assertEqual(len(e.open_spiders), 0)
class StopDownloadEngineTest(EngineTest):
@defer.inlineCallbacks
def test_crawler(self):
for spider in TestSpider, DictItemsSpider:
self.run = StopDownloadCrawlerRun(spider)
with LogCapture() as log:
yield self.run.run()
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
"Download stopped for <GET http://localhost:{}/redirected> from signal handler"
" StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
"Download stopped for <GET http://localhost:{}/> from signal handler"
" StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
"Download stopped for <GET http://localhost:{}/numbers> from signal handler"
" StopDownloadCrawlerRun.bytes_received".format(self.run.portno)))
self._assert_visited_urls()
self._assert_scheduled_requests(urls_to_visit=9)
self._assert_downloaded_responses()
self._assert_signals_caught()
self._assert_bytes_received()
def _assert_bytes_received(self):
self.assertEqual(9, len(self.run.bytes))
for request, data in self.run.bytes.items():
joined_data = b"".join(data)
self.assertTrue(len(data) == 1) # signal was fired only once
if self.run.getpath(request.url) == "/numbers":
# Received bytes are not the complete response. The exact amount depends
# on the buffer size, which can vary, so we only check that the amount
# of received bytes is strictly less than the full response.
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.assertTrue(len(joined_data) < len(b"".join(numbers)))
if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
start_test_site(debug=True)

113
tests/test_feedexport.py
View File

@ -9,6 +9,7 @@ import warnings
from abc import ABC, abstractmethod
from collections import defaultdict
from io import BytesIO
from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
from unittest import mock
@ -16,9 +17,11 @@ from urllib.parse import urljoin, urlparse, quote
from urllib.request import pathname2url
import lxml.etree
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial import unittest
from w3lib.url import path_to_file_uri
from w3lib.url import file_uri_to_path, path_to_file_uri
from zope.interface import implementer
from zope.interface.verify import verifyObject
import scrapy
@ -395,6 +398,46 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
pass
class DummyBlockingFeedStorage(BlockingFeedStorage):
def __init__(self, uri):
self.path = file_uri_to_path(uri)
def _store_in_thread(self, file):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
with open(self.path, 'ab') as output_file:
output_file.write(file.read())
file.close()
class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
def _store_in_thread(self, file):
raise OSError('Cannot store')
@implementer(IFeedStorage)
class LogOnStoreFileStorage:
"""
This storage logs inside `store` method.
It can be used to make sure `store` method is invoked.
"""
def __init__(self, uri):
self.path = file_uri_to_path(uri)
self.logger = getLogger()
def open(self, spider):
return tempfile.NamedTemporaryFile(prefix='feed-')
def store(self, file):
self.logger.info('Storage.store is called')
file.close()
class FeedExportTestBase(ABC, unittest.TestCase):
__test__ = False
@ -483,6 +526,7 @@ class FeedExportTest(FeedExportTestBase):
urljoin('file:', pathname2url(str(file_path))): feed
for file_path, feed in FEEDS.items()
}
content = {}
try:
with MockServer() as s:
@ -491,11 +535,17 @@ class FeedExportTest(FeedExportTestBase):
yield runner.crawl(spider_cls)
for file_path, feed in FEEDS.items():
if not os.path.exists(str(file_path)):
continue
with open(str(file_path), 'rb') as f:
content[feed['format']] = f.read()
finally:
for file_path in FEEDS.keys():
if not os.path.exists(str(file_path)):
continue
os.remove(str(file_path))
return content
@ -638,6 +688,25 @@ class FeedExportTest(FeedExportTestBase):
data = yield self.exported_no_data(settings)
self.assertEqual(expctd, data[fmt])
@defer.inlineCallbacks
def test_export_no_items_multiple_feeds(self):
""" Make sure that `storage.store` is called for every feed. """
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'json'},
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
'FEED_STORE_EMPTY': False
}
with LogCapture() as log:
yield self.exported_no_data(settings)
print(log)
self.assertEqual(str(log).count('Storage.store is called'), 3)
@defer.inlineCallbacks
def test_export_multiple_item_classes(self):
@ -994,6 +1063,48 @@ class FeedExportTest(FeedExportTestBase):
data = yield self.exported_no_data(settings)
self.assertEqual(data['csv'], b'')
@defer.inlineCallbacks
def test_multiple_feeds_success_logs_blocking_feed_storage(self):
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'json'},
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
}
items = [
{'foo': 'bar1', 'baz': ''},
{'foo': 'bar2', 'baz': 'quux'},
]
with LogCapture() as log:
yield self.exported_data(items, settings)
print(log)
for fmt in ['json', 'xml', 'csv']:
self.assertIn('Stored %s feed (2 items)' % fmt, str(log))
@defer.inlineCallbacks
def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'json'},
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
}
items = [
{'foo': 'bar1', 'baz': ''},
{'foo': 'bar2', 'baz': 'quux'},
]
with LogCapture() as log:
yield self.exported_data(items, settings)
print(log)
for fmt in ['json', 'xml', 'csv']:
self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True

21
tests/test_http_response.py
View File

@ -1,4 +1,5 @@
import unittest
from unittest import mock
from warnings import catch_warnings
from w3lib.encoding import resolve_encoding
@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
def test_json_response(self):
json_body = b"""{"ip": "109.187.217.200"}"""
json_response = self.response_class("http://www.example.com", body=json_body)
self.assertEqual(json_response.json(), {'ip': '109.187.217.200'})
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with self.assertRaises(ValueError):
text_response.json()
def test_cache_json_response(self):
json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
for json_body in json_valid_bodies:
json_response = self.response_class("http://www.example.com", body=json_body)
with mock.patch('json.loads') as mock_json:
for _ in range(2):
json_response.json()
mock_json.assert_called_once_with(json_body.decode())
class HtmlResponseTest(TextResponseTest):

47
tests/test_loader.py
View File

@ -1,6 +1,9 @@
from functools import partial
import unittest
import attr
from itemadapter import ItemAdapter
from scrapy.http import HtmlResponse
from scrapy.item import Item, Field
from scrapy.loader import ItemLoader
@ -9,6 +12,13 @@ from scrapy.loader.processors import (Compose, Identity, Join,
from scrapy.selector import Selector
try:
from dataclasses import make_dataclass, field as dataclass_field
except ImportError:
make_dataclass = None
dataclass_field = None
# test items
class NameItem(Item):
name = Field()
@ -28,6 +38,11 @@ class TestNestedItem(Item):
image = Field()
@attr.s
class AttrsNameItem:
name = attr.ib(default="")
# test item loaders
class NameItemLoader(ItemLoader):
default_item_class = TestItem
@ -466,7 +481,7 @@ class InitializationTestMixin:
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']})
def test_keep_list(self):
"""Loaded item should contain values from the initial item"""
@ -474,7 +489,7 @@ class InitializationTestMixin:
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
def test_add_value_singlevalue_singlevalue(self):
"""Values added after initialization should be appended"""
@ -483,7 +498,7 @@ class InitializationTestMixin:
il.add_value('name', 'bar')
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
def test_add_value_singlevalue_list(self):
"""Values added after initialization should be appended"""
@ -492,7 +507,7 @@ class InitializationTestMixin:
il.add_value('name', ['item', 'loader'])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']})
def test_add_value_list_singlevalue(self):
"""Values added after initialization should be appended"""
@ -501,7 +516,7 @@ class InitializationTestMixin:
il.add_value('name', 'qwerty')
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']})
def test_add_value_list_list(self):
"""Values added after initialization should be appended"""
@ -510,7 +525,7 @@ class InitializationTestMixin:
il.add_value('name', ['item', 'loader'])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']})
def test_get_output_value_singlevalue(self):
"""Getting output value must not remove value from item"""
@ -519,7 +534,7 @@ class InitializationTestMixin:
self.assertEqual(il.get_output_value('name'), ['foo'])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(loaded_item, dict({'name': ['foo']}))
self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']}))
def test_get_output_value_list(self):
"""Getting output value must not remove value from item"""
@ -528,7 +543,7 @@ class InitializationTestMixin:
self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']}))
def test_values_single(self):
"""Values from initial item must be added to loader._values"""
@ -551,6 +566,22 @@ class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
item_class = NameItem
class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase):
item_class = AttrsNameItem
@unittest.skipIf(not make_dataclass, "dataclasses module is not available")
class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if make_dataclass:
self.item_class = make_dataclass(
"TestDataClass",
[("name", list, dataclass_field(default_factory=list))],
)
class BaseNoInputReprocessingLoader(ItemLoader):
title_in = MapCompose(str.upper)
title_out = TakeFirst()

144
tests/test_pipeline_files.py
View File

@ -2,22 +2,41 @@ import os
import random
import time
from io import BytesIO
from tempfile import mkdtemp
from shutil import rmtree
from unittest import mock
from tempfile import mkdtemp
from unittest import mock, skipIf
from urllib.parse import urlparse
from twisted.trial import unittest
import attr
from itemadapter import ItemAdapter
from twisted.internet import defer
from twisted.trial import unittest
from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import (
FilesPipeline,
FSFilesStore,
FTPFilesStore,
GCSFilesStore,
S3FilesStore,
)
from scrapy.settings import Settings
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
from scrapy.utils.test import get_ftp_content_and_delete
from scrapy.utils.boto import is_botocore
from scrapy.utils.test import (
assert_aws_environ,
assert_gcs_environ,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
get_s3_content_and_delete,
)
try:
from dataclasses import make_dataclass, field as dataclass_field
except ImportError:
make_dataclass = None
dataclass_field = None
def _mocked_download_func(request, info):
@ -143,43 +162,88 @@ class FilesPipelineTestCase(unittest.TestCase):
p.stop()
class FilesPipelineTestCaseFields(unittest.TestCase):
class FilesPipelineTestCaseFieldsMixin:
def test_item_fields_default(self):
class TestItem(Item):
name = Field()
file_urls = Field()
files = Field()
for cls in TestItem, dict:
url = 'http://www.example.com/files/1.txt'
item = cls({'name': 'item1', 'file_urls': [url]})
pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
pipeline.item_completed(results, item, None)
self.assertEqual(item['files'], [results[0][1]])
url = 'http://www.example.com/files/1.txt'
item = self.item_class(name='item1', file_urls=[url])
pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
item = pipeline.item_completed(results, item, None)
files = ItemAdapter(item).get("files")
self.assertEqual(files, [results[0][1]])
self.assertIsInstance(item, self.item_class)
def test_item_fields_override_settings(self):
class TestItem(Item):
name = Field()
files = Field()
stored_file = Field()
url = 'http://www.example.com/files/1.txt'
item = self.item_class(name='item1', custom_file_urls=[url])
pipeline = FilesPipeline.from_settings(Settings({
'FILES_STORE': 's3://example/files/',
'FILES_URLS_FIELD': 'custom_file_urls',
'FILES_RESULT_FIELD': 'custom_files'
}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
item = pipeline.item_completed(results, item, None)
custom_files = ItemAdapter(item).get("custom_files")
self.assertEqual(custom_files, [results[0][1]])
self.assertIsInstance(item, self.item_class)
for cls in TestItem, dict:
url = 'http://www.example.com/files/1.txt'
item = cls({'name': 'item1', 'files': [url]})
pipeline = FilesPipeline.from_settings(Settings({
'FILES_STORE': 's3://example/files/',
'FILES_URLS_FIELD': 'files',
'FILES_RESULT_FIELD': 'stored_file'
}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
pipeline.item_completed(results, item, None)
self.assertEqual(item['stored_file'], [results[0][1]])
class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = dict
class FilesPipelineTestItem(Item):
name = Field()
# default fields
file_urls = Field()
files = Field()
# overridden fields
custom_file_urls = Field()
custom_files = Field()
class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = FilesPipelineTestItem
@skipIf(not make_dataclass, "dataclasses module is not available")
class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if make_dataclass:
self.item_class = make_dataclass(
"FilesPipelineTestDataClass",
[
("name", str),
# default fields
("file_urls", list, dataclass_field(default_factory=list)),
("files", list, dataclass_field(default_factory=list)),
# overridden fields
("custom_file_urls", list, dataclass_field(default_factory=list)),
("custom_files", list, dataclass_field(default_factory=list)),
],
)
@attr.s
class FilesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
file_urls = attr.ib(default=lambda: [])
files = attr.ib(default=lambda: [])
# overridden fields
custom_file_urls = attr.ib(default=lambda: [])
custom_files = attr.ib(default=lambda: [])
class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = FilesPipelineTestAttrsItem
class FilesPipelineTestCaseCustomSettings(unittest.TestCase):

129
tests/test_pipeline_images.py
View File

@ -1,17 +1,28 @@
import io
import hashlib
import io
import random
from tempfile import mkdtemp
from shutil import rmtree
from tempfile import mkdtemp
from unittest import skipIf
import attr
from itemadapter import ItemAdapter
from twisted.trial import unittest
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.item import Field, Item
from scrapy.pipelines.images import ImagesPipeline
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
try:
from dataclasses import make_dataclass, field as dataclass_field
except ImportError:
make_dataclass = None
dataclass_field = None
skip = False
try:
from PIL import Image
@ -124,43 +135,89 @@ class DeprecatedImagesPipeline(ImagesPipeline):
return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
class ImagesPipelineTestCaseFields(unittest.TestCase):
class ImagesPipelineTestCaseFieldsMixin:
def test_item_fields_default(self):
class TestItem(Item):
name = Field()
image_urls = Field()
images = Field()
for cls in TestItem, dict:
url = 'http://www.example.com/images/1.jpg'
item = cls({'name': 'item1', 'image_urls': [url]})
pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
pipeline.item_completed(results, item, None)
self.assertEqual(item['images'], [results[0][1]])
url = 'http://www.example.com/images/1.jpg'
item = self.item_class(name='item1', image_urls=[url])
pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
item = pipeline.item_completed(results, item, None)
images = ItemAdapter(item).get("images")
self.assertEqual(images, [results[0][1]])
self.assertIsInstance(item, self.item_class)
def test_item_fields_override_settings(self):
class TestItem(Item):
name = Field()
image = Field()
stored_image = Field()
url = 'http://www.example.com/images/1.jpg'
item = self.item_class(name='item1', custom_image_urls=[url])
pipeline = ImagesPipeline.from_settings(Settings({
'IMAGES_STORE': 's3://example/images/',
'IMAGES_URLS_FIELD': 'custom_image_urls',
'IMAGES_RESULT_FIELD': 'custom_images'
}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
item = pipeline.item_completed(results, item, None)
custom_images = ItemAdapter(item).get("custom_images")
self.assertEqual(custom_images, [results[0][1]])
self.assertIsInstance(item, self.item_class)
for cls in TestItem, dict:
url = 'http://www.example.com/images/1.jpg'
item = cls({'name': 'item1', 'image': [url]})
pipeline = ImagesPipeline.from_settings(Settings({
'IMAGES_STORE': 's3://example/images/',
'IMAGES_URLS_FIELD': 'image',
'IMAGES_RESULT_FIELD': 'stored_image'
}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {'url': url})]
pipeline.item_completed(results, item, None)
self.assertEqual(item['stored_image'], [results[0][1]])
class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = dict
class ImagesPipelineTestItem(Item):
name = Field()
# default fields
image_urls = Field()
images = Field()
# overridden fields
custom_image_urls = Field()
custom_images = Field()
class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = ImagesPipelineTestItem
@skipIf(not make_dataclass, "dataclasses module is not available")
class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if make_dataclass:
self.item_class = make_dataclass(
"FilesPipelineTestDataClass",
[
("name", str),
# default fields
("image_urls", list, dataclass_field(default_factory=list)),
("images", list, dataclass_field(default_factory=list)),
# overridden fields
("custom_image_urls", list, dataclass_field(default_factory=list)),
("custom_images", list, dataclass_field(default_factory=list)),
],
)
@attr.s
class ImagesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
image_urls = attr.ib(default=lambda: [])
images = attr.ib(default=lambda: [])
# overridden fields
custom_image_urls = attr.ib(default=lambda: [])
custom_images = attr.ib(default=lambda: [])
class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
item_class = ImagesPipelineTestAttrsItem
class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):

2
tests/test_scheduler.py
View File

@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
def __init__(self, start_urls):
self.start_urls = start_urls
super(StartUrlsSpider, self).__init__(start_urls)
super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
def parse(self, response):
pass

1
tests/test_utils_datatypes.py
View File

@ -271,6 +271,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
self.assertNotIn(r1, cache)
self.assertIn(r2, cache)
self.assertIn(r3, cache)
self.assertEqual(cache[r1], None)
self.assertEqual(cache[r2], 2)
self.assertEqual(cache[r3], 3)
del r2

43
tests/test_utils_serialize.py
View File

@ -1,18 +1,25 @@
import datetime
import json
import unittest
import datetime
from decimal import Decimal
import attr
from twisted.internet import defer
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.http import Request, Response
from scrapy.utils.serialize import ScrapyJSONEncoder
try:
from dataclasses import make_dataclass
except ImportError:
make_dataclass = None
class JsonEncoderTestCase(unittest.TestCase):
def setUp(self):
self.encoder = ScrapyJSONEncoder()
self.encoder = ScrapyJSONEncoder(sort_keys=True)
def test_encode_decode(self):
dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
@ -31,7 +38,8 @@ class JsonEncoderTestCase(unittest.TestCase):
for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts),
(dec, decs), (['foo', d], ['foo', ds]), (s, ss),
(dt_set, dt_sets)]:
self.assertEqual(self.encoder.encode(input), json.dumps(output))
self.assertEqual(self.encoder.encode(input),
json.dumps(output, sort_keys=True))
def test_encode_deferred(self):
self.assertIn('Deferred', self.encoder.encode(defer.Deferred()))
@ -47,3 +55,30 @@ class JsonEncoderTestCase(unittest.TestCase):
rs = self.encoder.encode(r)
self.assertIn(r.url, rs)
self.assertIn(str(r.status), rs)
@unittest.skipIf(not make_dataclass, "No dataclass support")
def test_encode_dataclass_item(self):
TestDataClass = make_dataclass(
"TestDataClass",
[("name", str), ("url", str), ("price", int)],
)
item = TestDataClass(name="Product", url="http://product.org", price=1)
encoded = self.encoder.encode(item)
self.assertEqual(
encoded,
'{"name": "Product", "price": 1, "url": "http://product.org"}'
)
def test_encode_attrs_item(self):
@attr.s
class AttrsItem:
name = attr.ib(type=str)
url = attr.ib(type=str)
price = attr.ib(type=int)
item = AttrsItem(name="Product", url="http://product.org", price=1)
encoded = self.encoder.encode(item)
self.assertEqual(
encoded,
'{"name": "Product", "price": 1, "url": "http://product.org"}'
)

10
tox.ini
View File

@ -24,6 +24,13 @@ passenv =
commands =
py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
[testenv:typing]
basepython = python3
deps =
mypy==0.780
commands =
mypy {posargs: scrapy tests}
[testenv:security]
basepython = python3
deps =
@ -38,7 +45,7 @@ deps =
pytest-flake8
commands =
py.test --flake8 {posargs:docs scrapy tests}
[testenv:pylint]
basepython = python3
deps =
@ -63,6 +70,7 @@ deps =
-ctests/constraints.txt
cryptography==2.0
cssselect==0.9.1
itemadapter==0.1.0
lxml==3.5.0
parsel==1.5.0
Protego==0.1.15