mirror of https://github.com/scrapy/scrapy.git
Ammar Najjar
parent
8cb53441b0
commit
68a7d05ed8
|
|
@ -237,7 +237,7 @@ coverage_ignore_pyobjects = [
|
|||
r'\bContractsManager\b$',
|
||||
|
||||
# For default contracts we only want to document their general purpose in
|
||||
# their constructor, the methods they reimplement to achieve that purpose
|
||||
# their __init__ method, the methods they reimplement to achieve that purpose
|
||||
# should be irrelevant to developers using those contracts.
|
||||
r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
|
||||
|
||||
|
|
|
|||
|
|
@ -84,12 +84,12 @@ New features
|
|||
convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
|
||||
|
||||
* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
|
||||
constructor now receives the :class:`~scrapy.http.Response` object that
|
||||
``__init__`` method now receives the :class:`~scrapy.http.Response` object that
|
||||
originated the request as its second argument (:issue:`3682`)
|
||||
|
||||
* A new ``restrict_text`` parameter for the
|
||||
:attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
constructor allows filtering links by linking text (:issue:`3622`,
|
||||
``__init__`` method allows filtering links by linking text (:issue:`3622`,
|
||||
:issue:`3635`)
|
||||
|
||||
* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
|
||||
|
|
@ -255,7 +255,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.selector.Selector`:
|
||||
|
||||
* ``_root`` (both the constructor argument and the object property, use
|
||||
* ``_root`` (both the ``__init__`` method argument and the object property, use
|
||||
``root``)
|
||||
|
||||
* ``extract_unquoted`` (use ``getall``)
|
||||
|
|
@ -2479,7 +2479,7 @@ Scrapy changes:
|
|||
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
|
||||
- promoted :ref:`topics-djangoitem` to main contrib
|
||||
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
|
||||
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- removed signal: ``scrapy.mail.mail_sent``
|
||||
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
|
||||
|
|
@ -2693,7 +2693,7 @@ API changes
|
|||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Quick example
|
|||
=============
|
||||
|
||||
There are two ways to instantiate the mail sender. You can instantiate it using
|
||||
the standard constructor::
|
||||
the standard ``__init__`` method::
|
||||
|
||||
from scrapy.mail import MailSender
|
||||
mailer = MailSender()
|
||||
|
|
@ -111,7 +111,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
Mail settings
|
||||
=============
|
||||
|
||||
These settings define the default constructor values of the :class:`MailSender`
|
||||
These settings define the default ``__init__`` method values of the :class:`MailSender`
|
||||
class, and can be used to configure e-mail notifications in your project without
|
||||
writing any code (for those extensions and code that uses :class:`MailSender`).
|
||||
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ described next.
|
|||
1. Declaring a serializer in the field
|
||||
--------------------------------------
|
||||
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
If you use :class:`~.Item` you can declare a serializer in the
|
||||
:ref:`field metadata <topics-items-fields>`. The serializer must be
|
||||
a callable which receives a value and returns its serialized form.
|
||||
|
||||
Example::
|
||||
|
|
@ -144,7 +144,7 @@ BaseItemExporter
|
|||
defining what fields to export, whether to export empty fields, or which
|
||||
encoding to use.
|
||||
|
||||
These features can be configured through the constructor arguments which
|
||||
These features can be configured through the `__init__` method arguments which
|
||||
populate their respective instance attributes: :attr:`fields_to_export`,
|
||||
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
|
||||
|
||||
|
|
@ -246,8 +246,8 @@ XmlItemExporter
|
|||
:param item_element: The name of each item element in the exported XML.
|
||||
:type item_element: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this `__init__` method are passed to the
|
||||
:class:`BaseItemExporter` `__init__` method
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -306,9 +306,9 @@ CsvItemExporter
|
|||
multi-valued fields, if found.
|
||||
:type include_headers_line: str
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor, and the leftover arguments to the
|
||||
`csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
|
||||
The additional keyword arguments of this `__init__` method are passed to the
|
||||
:class:`BaseItemExporter` `__init__` method, and the leftover arguments to the
|
||||
`csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -334,8 +334,8 @@ PickleItemExporter
|
|||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this `__init__` method are passed to the
|
||||
:class:`BaseItemExporter` `__init__` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
|
|
@ -351,8 +351,8 @@ PprintItemExporter
|
|||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
The additional keyword arguments of this `__init__` method are passed to the
|
||||
:class:`BaseItemExporter` `__init__` method
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -367,10 +367,10 @@ JsonItemExporter
|
|||
.. class:: JsonItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional constructor arguments are
|
||||
passed to the :class:`BaseItemExporter` constructor, and the leftover
|
||||
arguments to the `JSONEncoder`_ constructor, so you can use any
|
||||
`JSONEncoder`_ constructor argument to customize this exporter.
|
||||
objects as a list of objects. The additional `__init__` method arguments are
|
||||
passed to the :class:`BaseItemExporter` `__init__` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ `__init__` method, so you can use any
|
||||
`JSONEncoder`_ `__init__` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -398,10 +398,10 @@ JsonLinesItemExporter
|
|||
.. class:: JsonLinesItemExporter(file, \**kwargs)
|
||||
|
||||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional constructor arguments are passed
|
||||
to the :class:`BaseItemExporter` constructor, and the leftover arguments to
|
||||
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
|
||||
constructor argument to customize this exporter.
|
||||
JSON-encoded item per line. The additional `__init__` method arguments are passed
|
||||
to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to
|
||||
the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_
|
||||
`__init__` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ Loading & activating extensions
|
|||
|
||||
Extensions are loaded and activated at startup by instantiating a single
|
||||
instance of the extension class. Therefore, all the extension initialization
|
||||
code must be performed in the class constructor (``__init__`` method).
|
||||
code must be performed in the class ``__init__`` method.
|
||||
|
||||
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
|
||||
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ especially in a larger project with many spiders.
|
|||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
serialization can be customized using Item fields metadata, :mod:`trackref`
|
||||
tracks Item instances to help find memory leaks
|
||||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
|
@ -237,7 +237,7 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its constructor, and
|
||||
Items replicate the standard `dict API`_, including its `__init__` method and
|
||||
also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Using Item Loaders to populate items
|
|||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
|
||||
which case an Item is automatically instantiated in the Item Loader constructor
|
||||
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
|
||||
using the Item class specified in the :attr:`ItemLoader.default_item_class`
|
||||
attribute.
|
||||
|
||||
|
|
@ -265,7 +265,7 @@ There are several ways to modify Item Loader context values:
|
|||
loader.context['unit'] = 'cm'
|
||||
|
||||
2. On Item Loader instantiation (the keyword arguments of Item Loader
|
||||
constructor are stored in the Item Loader context)::
|
||||
``__init__`` methodare stored in the Item Loader context)::
|
||||
|
||||
loader = ItemLoader(product, unit='cm')
|
||||
|
||||
|
|
@ -494,7 +494,7 @@ ItemLoader objects
|
|||
.. attribute:: default_item_class
|
||||
|
||||
An Item class (or factory), used to instantiate items when not given in
|
||||
the constructor.
|
||||
the `__init__` method
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
|
|
@ -509,15 +509,15 @@ ItemLoader objects
|
|||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the constructor.
|
||||
If a selector is given in the constructor this attribute is ignored.
|
||||
:class:`ItemLoader`, if only a response is given in the `__init__` method
|
||||
If a selector is given in the `__init__` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the constructor or one created from
|
||||
the response given in the constructor using the
|
||||
It's either the selector given in the `__init__` methodor one created from
|
||||
the response given in the `__init__` methodusing the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
|
||||
|
|
@ -642,7 +642,7 @@ Here is a list of all built-in processors:
|
|||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any constructor arguments, nor does it
|
||||
values unchanged. It doesn't receive any `__init__` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
|
@ -656,7 +656,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any constructor arguments, nor does it accept Loader contexts.
|
||||
It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts.
|
||||
|
||||
Example::
|
||||
|
||||
|
|
@ -667,7 +667,7 @@ Here is a list of all built-in processors:
|
|||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the constructor, which
|
||||
Returns the values joined with the separator given in the `__init__` method which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
|
|
@ -705,7 +705,7 @@ Here is a list of all built-in processors:
|
|||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the constructor are used as the default
|
||||
The keyword arguments passed in the `__init__` methodare used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
|
|
@ -749,12 +749,12 @@ Here is a list of all built-in processors:
|
|||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
constructor keyword arguments are used as default context values. See
|
||||
`__init__` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the constructor and returns the output.
|
||||
Queries the value using the json path provided to the `__init__` methodand returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ Request objects
|
|||
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
attribute contains the escaped URL, so it can differ from the URL passed in
|
||||
the constructor.
|
||||
the `__init__` method
|
||||
|
||||
This attribute is read-only. To change the URL of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
.. class:: FormRequest(url, [formdata, ...])
|
||||
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
|
||||
The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
|
|||
:type dont_click: boolean
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` constructor.
|
||||
:class:`FormRequest` `__init__` method
|
||||
|
||||
.. versionadded:: 0.10.3
|
||||
The ``formname`` parameter.
|
||||
|
|
@ -547,7 +547,7 @@ dealing with JSON requests.
|
|||
|
||||
.. class:: JsonRequest(url, [... data, dumps_kwargs])
|
||||
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
|
||||
The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The
|
||||
remaining arguments are the same as for the :class:`Request` class and are
|
||||
not documented here.
|
||||
|
||||
|
|
@ -556,7 +556,7 @@ dealing with JSON requests.
|
|||
|
||||
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
|
||||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
|
|
@ -721,7 +721,7 @@ TextResponse objects
|
|||
:class:`Response` class, which is meant to be used only for binary data,
|
||||
such as images, sounds or any media file.
|
||||
|
||||
:class:`TextResponse` objects support a new constructor argument, in
|
||||
:class:`TextResponse` objects support a new `__init__` method argument, in
|
||||
addition to the base :class:`Response` objects. The remaining functionality
|
||||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
|
|
@ -755,7 +755,7 @@ TextResponse objects
|
|||
A string with the encoding of this response. The encoding is resolved by
|
||||
trying the following mechanisms, in order:
|
||||
|
||||
1. the encoding passed in the constructor ``encoding`` argument
|
||||
1. the encoding passed in the `__init__` method`` ``encoding`` argument
|
||||
|
||||
2. the encoding declared in the Content-Type HTTP header. If this
|
||||
encoding is not valid (ie. unknown), it is ignored and the next
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class BaseItemExporter(object):
|
|||
def _configure(self, options, dont_fail=False):
|
||||
"""Configure the exporter by poping options from the ``options`` dict.
|
||||
If dont_fail is set, it won't raise an exception on unexpected options
|
||||
(useful for using with keyword arguments in subclasses constructors)
|
||||
(useful for using with keyword arguments in subclasses __init__ methods)
|
||||
"""
|
||||
self.encoding = options.pop('encoding', None)
|
||||
self.fields_to_export = options.pop('fields_to_export', None)
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
warnings.warn(
|
||||
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
|
||||
"without AWS keys is deprecated. Please supply credentials or "
|
||||
"use the `from_crawler()` constructor.",
|
||||
"use the `from_crawler()` __init__ method.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
|
|
|||
|
|
@ -246,7 +246,7 @@ class CaselessDict(dict):
|
|||
class MergeDict(object):
|
||||
"""
|
||||
A simple class for creating new "virtual" dictionaries that actually look
|
||||
up values in more than one dictionary, passed in the constructor.
|
||||
up values in more than one dictionary, passed in the __init__ method.
|
||||
|
||||
If a key appears in more than one of the given dictionaries, only the
|
||||
first occurrence will be used.
|
||||
|
|
|
|||
|
|
@ -123,14 +123,14 @@ def rel_has_nofollow(rel):
|
|||
|
||||
def create_instance(objcls, settings, crawler, *args, **kwargs):
|
||||
"""Construct a class instance using its ``from_crawler`` or
|
||||
``from_settings`` constructors, if available.
|
||||
``from_settings`` __init__ method, if available.
|
||||
|
||||
At least one of ``settings`` and ``crawler`` needs to be different from
|
||||
``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
|
||||
If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
|
||||
If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be
|
||||
tried.
|
||||
|
||||
``*args`` and ``**kwargs`` are forwarded to the constructors.
|
||||
``*args`` and ``**kwargs`` are forwarded to the __init__ methods.
|
||||
|
||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ class WeakKeyCache(object):
|
|||
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
||||
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
||||
False) of the given dict converted to strings. ``dct_or_tuples`` can be a
|
||||
dict or a list of tuples, like any dict constructor supports.
|
||||
dict or a list of tuples, like any dict __init__ method supports.
|
||||
"""
|
||||
d = {}
|
||||
for k, v in six.iteritems(dict(dct_or_tuples)):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ singletons members of that object, as explained below:
|
|||
``scrapy.core.manager.ExecutionManager``) - instantiated with a ``Settings``
|
||||
object
|
||||
|
||||
- **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the constructor)
|
||||
- **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the ``__init__`` method)
|
||||
- **crawler.extensions**: ``scrapy.extension.ExtensionManager`` instance
|
||||
- **crawler.engine**: ``scrapy.core.engine.ExecutionEngine`` instance
|
||||
- ``crawler.engine.scheduler``
|
||||
|
|
@ -55,7 +55,7 @@ singletons members of that object, as explained below:
|
|||
``STATS_CLASS`` setting)
|
||||
- **crawler.log**: Logger class with methods replacing the current
|
||||
``scrapy.log`` functions. Logging would be started (if enabled) on
|
||||
``Crawler`` constructor, so no log starting functions are required.
|
||||
``Crawler`` __init__ method, so no log starting functions are required.
|
||||
|
||||
- ``crawler.log.msg``
|
||||
- **crawler.signals**: signal handling
|
||||
|
|
@ -69,12 +69,12 @@ Required code changes after singletons removal
|
|||
==============================================
|
||||
|
||||
All components (extensions, middlewares, etc) will receive this ``Crawler``
|
||||
object in their constructors, and this will be the only mechanism for accessing
|
||||
object in their ``__init__`` methods, and this will be the only mechanism for accessing
|
||||
any other components (as opposed to importing each singleton from their
|
||||
respective module). This will also serve to stabilize the core API, something
|
||||
which we haven't documented so far (partly because of this).
|
||||
|
||||
So, for a typical middleware constructor code, instead of this:
|
||||
So, for a typical middleware ``__init__`` method code, instead of this:
|
||||
|
||||
::
|
||||
|
||||
|
|
@ -125,13 +125,13 @@ Open issues to resolve
|
|||
|
||||
- Should we pass ``Settings`` object to ``ScrapyCommand.add_options()``?
|
||||
- How should spiders access settings?
|
||||
- Option 1. Pass ``Crawler`` object to spider constructors too
|
||||
- Option 1. Pass ``Crawler`` object to spider ``__init__`` methods too
|
||||
- pro: one way to access all components (settings and signals being the
|
||||
most relevant to spiders)
|
||||
- con?: spider code can access (and control) any crawler component -
|
||||
since we don't want to support spiders messing with the crawler (write
|
||||
an extension or spider middleware if you need that)
|
||||
- Option 2. Pass ``Settings`` object to spider constructors, which would
|
||||
- Option 2. Pass ``Settings`` object to spider ``__init__`` methods, which would
|
||||
then be accessed through ``self.settings``, like logging which is accessed
|
||||
through ``self.log``
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class RequestTest(unittest.TestCase):
|
|||
default_meta = {}
|
||||
|
||||
def test_init(self):
|
||||
# Request requires url in the constructor
|
||||
# Request requires url in the __init__ method
|
||||
self.assertRaises(Exception, self.request_class)
|
||||
|
||||
# url argument must be basestring
|
||||
|
|
@ -500,7 +500,7 @@ class FormRequestTest(RequestTest):
|
|||
formdata=(('foo', 'bar'), ('foo', 'baz')))
|
||||
self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
|
||||
self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
|
||||
|
||||
|
||||
def test_from_response_override_duplicate_form_key(self):
|
||||
response = _buildresponse(
|
||||
"""<form action="get.php" method="POST">
|
||||
|
|
@ -657,7 +657,7 @@ class FormRequestTest(RequestTest):
|
|||
req = self.request_class.from_response(response, dont_click=True)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']})
|
||||
|
||||
|
||||
def test_from_response_clickdata_does_not_ignore_image(self):
|
||||
response = _buildresponse(
|
||||
"""<form>
|
||||
|
|
|
|||
|
|
@ -534,7 +534,7 @@ class XmlResponseTest(TextResponseTest):
|
|||
r2 = self.response_class("http://www.example.com", body=body)
|
||||
self._assert_response_values(r2, 'iso-8859-1', body)
|
||||
|
||||
# make sure replace() preserves the explicit encoding passed in the constructor
|
||||
# make sure replace() preserves the explicit encoding passed in the __init__ method
|
||||
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
|
||||
r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8')
|
||||
body2 = b"New body"
|
||||
|
|
|
|||
|
|
@ -548,11 +548,11 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
</html>
|
||||
""")
|
||||
|
||||
def test_constructor(self):
|
||||
def test_init_method(self):
|
||||
l = TestItemLoader()
|
||||
self.assertEqual(l.selector, None)
|
||||
|
||||
def test_constructor_errors(self):
|
||||
def test_init_method_errors(self):
|
||||
l = TestItemLoader()
|
||||
self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href')
|
||||
self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href')
|
||||
|
|
@ -561,7 +561,7 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text')
|
||||
self.assertRaises(RuntimeError, l.get_css, '#name::text')
|
||||
|
||||
def test_constructor_with_selector(self):
|
||||
def test_init_method_with_selector(self):
|
||||
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
|
||||
l = TestItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
|
|
@ -569,7 +569,7 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
|
||||
def test_constructor_with_selector_css(self):
|
||||
def test_init_method_with_selector_css(self):
|
||||
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
|
||||
l = TestItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
|
|
@ -577,14 +577,14 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
l.add_css('name', 'div::text')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
|
||||
def test_constructor_with_response(self):
|
||||
def test_init_method_with_response(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
|
||||
def test_constructor_with_response_css(self):
|
||||
def test_init_method_with_response_css(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
|
||||
|
|
|
|||
|
|
@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase):
|
|||
self.assertEqual(list(start_requests), [])
|
||||
|
||||
def test_spider_args(self):
|
||||
"""Constructor arguments are assigned to spider attributes"""
|
||||
"""__init__ method arguments are assigned to spider attributes"""
|
||||
spider = self.spider_class('example.com', foo='bar')
|
||||
self.assertEqual(spider.foo, 'bar')
|
||||
|
||||
def test_spider_without_name(self):
|
||||
"""Constructor arguments are assigned to spider attributes"""
|
||||
"""__init__ method arguments are assigned to spider attributes"""
|
||||
self.assertRaises(ValueError, self.spider_class)
|
||||
self.assertRaises(ValueError, self.spider_class, somearg='foo')
|
||||
|
||||
|
|
|
|||
|
|
@ -109,11 +109,11 @@ class UtilsMiscTestCase(unittest.TestCase):
|
|||
else:
|
||||
mock.assert_called_once_with(*args, **kwargs)
|
||||
|
||||
# Check usage of correct constructor using four mocks:
|
||||
# 1. with no alternative constructors
|
||||
# 2. with from_settings() constructor
|
||||
# 3. with from_crawler() constructor
|
||||
# 4. with from_settings() and from_crawler() constructor
|
||||
# Check usage of correct __init__ method using four mocks:
|
||||
# 1. with no alternative __init__ methods
|
||||
# 2. with from_settings() __init__ method
|
||||
# 3. with from_crawler() __init__ method
|
||||
# 4. with from_settings() and from_crawler() __init__ method
|
||||
spec_sets = ([], ['from_settings'], ['from_crawler'],
|
||||
['from_settings', 'from_crawler'])
|
||||
for specs in spec_sets:
|
||||
|
|
|
|||
Loading…
Reference in New Issue