mirror of https://github.com/scrapy/scrapy.git
parent
68a7d05ed8
commit
da8cd9448d
|
|
@ -144,7 +144,7 @@ BaseItemExporter
|
||||||
defining what fields to export, whether to export empty fields, or which
|
defining what fields to export, whether to export empty fields, or which
|
||||||
encoding to use.
|
encoding to use.
|
||||||
|
|
||||||
These features can be configured through the `__init__` method arguments which
|
These features can be configured through the ``__init__`` method arguments which
|
||||||
populate their respective instance attributes: :attr:`fields_to_export`,
|
populate their respective instance attributes: :attr:`fields_to_export`,
|
||||||
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
||||||
|
|
||||||
|
|
@ -246,8 +246,8 @@ XmlItemExporter
|
||||||
:param item_element: The name of each item element in the exported XML.
|
:param item_element: The name of each item element in the exported XML.
|
||||||
:type item_element: str
|
:type item_element: str
|
||||||
|
|
||||||
The additional keyword arguments of this `__init__` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` `__init__` method
|
:class:`BaseItemExporter` ``__init__`` method
|
||||||
|
|
||||||
A typical output of this exporter would be::
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
|
|
@ -306,9 +306,9 @@ CsvItemExporter
|
||||||
multi-valued fields, if found.
|
multi-valued fields, if found.
|
||||||
:type include_headers_line: str
|
:type include_headers_line: str
|
||||||
|
|
||||||
The additional keyword arguments of this `__init__` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` `__init__` method, and the leftover arguments to the
|
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||||
`csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method
|
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||||
argument to customize this exporter.
|
argument to customize this exporter.
|
||||||
|
|
||||||
A typical output of this exporter would be::
|
A typical output of this exporter would be::
|
||||||
|
|
@ -334,8 +334,8 @@ PickleItemExporter
|
||||||
|
|
||||||
For more information, refer to the `pickle module documentation`_.
|
For more information, refer to the `pickle module documentation`_.
|
||||||
|
|
||||||
The additional keyword arguments of this `__init__` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` `__init__` method.
|
:class:`BaseItemExporter` ``__init__`` method.
|
||||||
|
|
||||||
Pickle isn't a human readable format, so no output examples are provided.
|
Pickle isn't a human readable format, so no output examples are provided.
|
||||||
|
|
||||||
|
|
@ -351,8 +351,8 @@ PprintItemExporter
|
||||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||||
|
|
||||||
The additional keyword arguments of this `__init__` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` `__init__` method
|
:class:`BaseItemExporter` ``__init__`` method
|
||||||
|
|
||||||
A typical output of this exporter would be::
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
|
|
@ -367,10 +367,10 @@ JsonItemExporter
|
||||||
.. class:: JsonItemExporter(file, \**kwargs)
|
.. class:: JsonItemExporter(file, \**kwargs)
|
||||||
|
|
||||||
Exports Items in JSON format to the specified file-like object, writing all
|
Exports Items in JSON format to the specified file-like object, writing all
|
||||||
objects as a list of objects. The additional `__init__` method arguments are
|
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||||
passed to the :class:`BaseItemExporter` `__init__` method, and the leftover
|
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||||
arguments to the `JSONEncoder`_ `__init__` method, so you can use any
|
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||||
`JSONEncoder`_ `__init__` method argument to customize this exporter.
|
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||||
|
|
||||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||||
|
|
@ -398,10 +398,10 @@ JsonLinesItemExporter
|
||||||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||||
|
|
||||||
Exports Items in JSON format to the specified file-like object, writing one
|
Exports Items in JSON format to the specified file-like object, writing one
|
||||||
JSON-encoded item per line. The additional `__init__` method arguments are passed
|
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||||
to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to
|
to the :class:`BaseItemExporter` ``__init__`` method and the leftover arguments to
|
||||||
the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_
|
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||||
`__init__` method argument to customize this exporter.
|
``__init__`` method argument to customize this exporter.
|
||||||
|
|
||||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||||
|
|
|
||||||
|
|
@ -237,7 +237,7 @@ Item objects
|
||||||
|
|
||||||
Return a new Item optionally initialized from the given argument.
|
Return a new Item optionally initialized from the given argument.
|
||||||
|
|
||||||
Items replicate the standard `dict API`_, including its `__init__` method and
|
Items replicate the standard `dict API`_, including its ``__init__`` method and
|
||||||
also provide the following additional API members:
|
also provide the following additional API members:
|
||||||
|
|
||||||
.. automethod:: copy
|
.. automethod:: copy
|
||||||
|
|
|
||||||
|
|
@ -494,7 +494,7 @@ ItemLoader objects
|
||||||
.. attribute:: default_item_class
|
.. attribute:: default_item_class
|
||||||
|
|
||||||
An Item class (or factory), used to instantiate items when not given in
|
An Item class (or factory), used to instantiate items when not given in
|
||||||
the `__init__` method
|
the ``__init__`` method
|
||||||
|
|
||||||
.. attribute:: default_input_processor
|
.. attribute:: default_input_processor
|
||||||
|
|
||||||
|
|
@ -509,15 +509,15 @@ ItemLoader objects
|
||||||
.. attribute:: default_selector_class
|
.. attribute:: default_selector_class
|
||||||
|
|
||||||
The class used to construct the :attr:`selector` of this
|
The class used to construct the :attr:`selector` of this
|
||||||
:class:`ItemLoader`, if only a response is given in the `__init__` method
|
:class:`ItemLoader`, if only a response is given in the ``__init__`` method
|
||||||
If a selector is given in the `__init__` method this attribute is ignored.
|
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||||
This attribute is sometimes overridden in subclasses.
|
This attribute is sometimes overridden in subclasses.
|
||||||
|
|
||||||
.. attribute:: selector
|
.. attribute:: selector
|
||||||
|
|
||||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||||
It's either the selector given in the `__init__` methodor one created from
|
It's either the selector given in the ``__init__`` methodor one created from
|
||||||
the response given in the `__init__` methodusing the
|
the response given in the ``__init__`` methodusing the
|
||||||
:attr:`default_selector_class`. This attribute is meant to be
|
:attr:`default_selector_class`. This attribute is meant to be
|
||||||
read-only.
|
read-only.
|
||||||
|
|
||||||
|
|
@ -642,7 +642,7 @@ Here is a list of all built-in processors:
|
||||||
.. class:: Identity
|
.. class:: Identity
|
||||||
|
|
||||||
The simplest processor, which doesn't do anything. It returns the original
|
The simplest processor, which doesn't do anything. It returns the original
|
||||||
values unchanged. It doesn't receive any `__init__` method arguments, nor does it
|
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||||
accept Loader contexts.
|
accept Loader contexts.
|
||||||
|
|
||||||
Example::
|
Example::
|
||||||
|
|
@ -656,7 +656,7 @@ Here is a list of all built-in processors:
|
||||||
|
|
||||||
Returns the first non-null/non-empty value from the values received,
|
Returns the first non-null/non-empty value from the values received,
|
||||||
so it's typically used as an output processor to single-valued fields.
|
so it's typically used as an output processor to single-valued fields.
|
||||||
It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts.
|
It doesn't receive any ``__init__`` methodarguments, nor does it accept Loader contexts.
|
||||||
|
|
||||||
Example::
|
Example::
|
||||||
|
|
||||||
|
|
@ -667,7 +667,7 @@ Here is a list of all built-in processors:
|
||||||
|
|
||||||
.. class:: Join(separator=u' ')
|
.. class:: Join(separator=u' ')
|
||||||
|
|
||||||
Returns the values joined with the separator given in the `__init__` method which
|
Returns the values joined with the separator given in the ``__init__`` method which
|
||||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||||
|
|
||||||
When using the default separator, this processor is equivalent to the
|
When using the default separator, this processor is equivalent to the
|
||||||
|
|
@ -705,7 +705,7 @@ Here is a list of all built-in processors:
|
||||||
those which do, this processor will pass the currently active :ref:`Loader
|
those which do, this processor will pass the currently active :ref:`Loader
|
||||||
context <topics-loaders-context>` through that parameter.
|
context <topics-loaders-context>` through that parameter.
|
||||||
|
|
||||||
The keyword arguments passed in the `__init__` methodare used as the default
|
The keyword arguments passed in the ``__init__`` methodare used as the default
|
||||||
Loader context values passed to each function call. However, the final
|
Loader context values passed to each function call. However, the final
|
||||||
Loader context values passed to functions are overridden with the currently
|
Loader context values passed to functions are overridden with the currently
|
||||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||||
|
|
@ -749,12 +749,12 @@ Here is a list of all built-in processors:
|
||||||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||||
|
|
||||||
As with the Compose processor, functions can receive Loader contexts, and
|
As with the Compose processor, functions can receive Loader contexts, and
|
||||||
`__init__` method keyword arguments are used as default context values. See
|
``__init__`` method keyword arguments are used as default context values. See
|
||||||
:class:`Compose` processor for more info.
|
:class:`Compose` processor for more info.
|
||||||
|
|
||||||
.. class:: SelectJmes(json_path)
|
.. class:: SelectJmes(json_path)
|
||||||
|
|
||||||
Queries the value using the json path provided to the `__init__` methodand returns the output.
|
Queries the value using the json path provided to the ``__init__`` methodand returns the output.
|
||||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||||
This processor takes only one input at a time.
|
This processor takes only one input at a time.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -137,7 +137,7 @@ Request objects
|
||||||
|
|
||||||
A string containing the URL of this request. Keep in mind that this
|
A string containing the URL of this request. Keep in mind that this
|
||||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||||
the `__init__` method
|
the ``__init__`` method
|
||||||
|
|
||||||
This attribute is read-only. To change the URL of a Request use
|
This attribute is read-only. To change the URL of a Request use
|
||||||
:meth:`replace`.
|
:meth:`replace`.
|
||||||
|
|
@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
|
||||||
|
|
||||||
.. class:: FormRequest(url, [formdata, ...])
|
.. class:: FormRequest(url, [formdata, ...])
|
||||||
|
|
||||||
The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The
|
The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method The
|
||||||
remaining arguments are the same as for the :class:`Request` class and are
|
remaining arguments are the same as for the :class:`Request` class and are
|
||||||
not documented here.
|
not documented here.
|
||||||
|
|
||||||
|
|
@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
|
||||||
:type dont_click: boolean
|
:type dont_click: boolean
|
||||||
|
|
||||||
The other parameters of this class method are passed directly to the
|
The other parameters of this class method are passed directly to the
|
||||||
:class:`FormRequest` `__init__` method
|
:class:`FormRequest` ``__init__`` method
|
||||||
|
|
||||||
.. versionadded:: 0.10.3
|
.. versionadded:: 0.10.3
|
||||||
The ``formname`` parameter.
|
The ``formname`` parameter.
|
||||||
|
|
@ -547,7 +547,7 @@ dealing with JSON requests.
|
||||||
|
|
||||||
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
||||||
|
|
||||||
The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The
|
The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method The
|
||||||
remaining arguments are the same as for the :class:`Request` class and are
|
remaining arguments are the same as for the :class:`Request` class and are
|
||||||
not documented here.
|
not documented here.
|
||||||
|
|
||||||
|
|
@ -721,7 +721,7 @@ TextResponse objects
|
||||||
:class:`Response` class, which is meant to be used only for binary data,
|
:class:`Response` class, which is meant to be used only for binary data,
|
||||||
such as images, sounds or any media file.
|
such as images, sounds or any media file.
|
||||||
|
|
||||||
:class:`TextResponse` objects support a new `__init__` method argument, in
|
:class:`TextResponse` objects support a new ``__init__`` method argument, in
|
||||||
addition to the base :class:`Response` objects. The remaining functionality
|
addition to the base :class:`Response` objects. The remaining functionality
|
||||||
is the same as for the :class:`Response` class and is not documented here.
|
is the same as for the :class:`Response` class and is not documented here.
|
||||||
|
|
||||||
|
|
@ -755,7 +755,7 @@ TextResponse objects
|
||||||
A string with the encoding of this response. The encoding is resolved by
|
A string with the encoding of this response. The encoding is resolved by
|
||||||
trying the following mechanisms, in order:
|
trying the following mechanisms, in order:
|
||||||
|
|
||||||
1. the encoding passed in the `__init__` method`` ``encoding`` argument
|
1. the encoding passed in the ``__init__`` method`` ``encoding`` argument
|
||||||
|
|
||||||
2. the encoding declared in the Content-Type HTTP header. If this
|
2. the encoding declared in the Content-Type HTTP header. If this
|
||||||
encoding is not valid (ie. unknown), it is ignored and the next
|
encoding is not valid (ie. unknown), it is ignored and the next
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ class BaseItemExporter(object):
|
||||||
def _configure(self, options, dont_fail=False):
|
def _configure(self, options, dont_fail=False):
|
||||||
"""Configure the exporter by poping options from the ``options`` dict.
|
"""Configure the exporter by poping options from the ``options`` dict.
|
||||||
If dont_fail is set, it won't raise an exception on unexpected options
|
If dont_fail is set, it won't raise an exception on unexpected options
|
||||||
(useful for using with keyword arguments in subclasses __init__ methods)
|
(useful for using with keyword arguments in subclasses ``__init__`` methods)
|
||||||
"""
|
"""
|
||||||
self.encoding = options.pop('encoding', None)
|
self.encoding = options.pop('encoding', None)
|
||||||
self.fields_to_export = options.pop('fields_to_export', None)
|
self.fields_to_export = options.pop('fields_to_export', None)
|
||||||
|
|
|
||||||
|
|
@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage):
|
||||||
warnings.warn(
|
warnings.warn(
|
||||||
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
|
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
|
||||||
"without AWS keys is deprecated. Please supply credentials or "
|
"without AWS keys is deprecated. Please supply credentials or "
|
||||||
"use the `from_crawler()` __init__ method.",
|
"use the `from_crawler()` ``__init__`` method.",
|
||||||
category=ScrapyDeprecationWarning,
|
category=ScrapyDeprecationWarning,
|
||||||
stacklevel=2
|
stacklevel=2
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -246,7 +246,7 @@ class CaselessDict(dict):
|
||||||
class MergeDict(object):
|
class MergeDict(object):
|
||||||
"""
|
"""
|
||||||
A simple class for creating new "virtual" dictionaries that actually look
|
A simple class for creating new "virtual" dictionaries that actually look
|
||||||
up values in more than one dictionary, passed in the __init__ method.
|
up values in more than one dictionary, passed in the ``__init__`` method.
|
||||||
|
|
||||||
If a key appears in more than one of the given dictionaries, only the
|
If a key appears in more than one of the given dictionaries, only the
|
||||||
first occurrence will be used.
|
first occurrence will be used.
|
||||||
|
|
|
||||||
|
|
@ -123,14 +123,14 @@ def rel_has_nofollow(rel):
|
||||||
|
|
||||||
def create_instance(objcls, settings, crawler, *args, **kwargs):
|
def create_instance(objcls, settings, crawler, *args, **kwargs):
|
||||||
"""Construct a class instance using its ``from_crawler`` or
|
"""Construct a class instance using its ``from_crawler`` or
|
||||||
``from_settings`` __init__ method, if available.
|
``from_settings`` ``__init__`` method, if available.
|
||||||
|
|
||||||
At least one of ``settings`` and ``crawler`` needs to be different from
|
At least one of ``settings`` and ``crawler`` needs to be different from
|
||||||
``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
|
``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
|
||||||
If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be
|
If ``crawler`` is ``None``, only the ``from_settings`` ``__init__`` method will be
|
||||||
tried.
|
tried.
|
||||||
|
|
||||||
``*args`` and ``**kwargs`` are forwarded to the __init__ methods.
|
``*args`` and ``**kwargs`` are forwarded to the ``__init__`` methods.
|
||||||
|
|
||||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -303,7 +303,7 @@ class WeakKeyCache(object):
|
||||||
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
||||||
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
||||||
False) of the given dict converted to strings. ``dct_or_tuples`` can be a
|
False) of the given dict converted to strings. ``dct_or_tuples`` can be a
|
||||||
dict or a list of tuples, like any dict __init__ method supports.
|
dict or a list of tuples, like any dict ``__init__`` method supports.
|
||||||
"""
|
"""
|
||||||
d = {}
|
d = {}
|
||||||
for k, v in six.iteritems(dict(dct_or_tuples)):
|
for k, v in six.iteritems(dict(dct_or_tuples)):
|
||||||
|
|
|
||||||
|
|
@ -55,7 +55,7 @@ singletons members of that object, as explained below:
|
||||||
``STATS_CLASS`` setting)
|
``STATS_CLASS`` setting)
|
||||||
- **crawler.log**: Logger class with methods replacing the current
|
- **crawler.log**: Logger class with methods replacing the current
|
||||||
``scrapy.log`` functions. Logging would be started (if enabled) on
|
``scrapy.log`` functions. Logging would be started (if enabled) on
|
||||||
``Crawler`` __init__ method, so no log starting functions are required.
|
``Crawler`` ``__init__`` method, so no log starting functions are required.
|
||||||
|
|
||||||
- ``crawler.log.msg``
|
- ``crawler.log.msg``
|
||||||
- **crawler.signals**: signal handling
|
- **crawler.signals**: signal handling
|
||||||
|
|
|
||||||
|
|
@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase):
|
||||||
self.assertEqual(list(start_requests), [])
|
self.assertEqual(list(start_requests), [])
|
||||||
|
|
||||||
def test_spider_args(self):
|
def test_spider_args(self):
|
||||||
"""__init__ method arguments are assigned to spider attributes"""
|
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||||
spider = self.spider_class('example.com', foo='bar')
|
spider = self.spider_class('example.com', foo='bar')
|
||||||
self.assertEqual(spider.foo, 'bar')
|
self.assertEqual(spider.foo, 'bar')
|
||||||
|
|
||||||
def test_spider_without_name(self):
|
def test_spider_without_name(self):
|
||||||
"""__init__ method arguments are assigned to spider attributes"""
|
"""``__init__`` method arguments are assigned to spider attributes"""
|
||||||
self.assertRaises(ValueError, self.spider_class)
|
self.assertRaises(ValueError, self.spider_class)
|
||||||
self.assertRaises(ValueError, self.spider_class, somearg='foo')
|
self.assertRaises(ValueError, self.spider_class, somearg='foo')
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue