RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into asyncio-startrequests-asyncgen

This commit is contained in:
Adrián Chaves 2020-08-10 13:52:06 +02:00 committed by GitHub
parent 914726f696 0cf1340c29
commit 43a53f3cbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
94 changed files with 1718 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.2.0
current_version = 2.3.0
commit = True
tag = True
tag_name = {new_version}

2
docs/_ext/scrapydocs.py
View File

@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
def is_setting_index(node):
if node.tagname == 'index':
# index entries for setting directives look like:
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
entry_type, info, refid = node['entries'][0][:3]
return entry_type == 'pair' and info.endswith('; setting')
return False

133
docs/news.rst
View File

@ -3,6 +3,139 @@
Release notes
=============
.. _release-2.3.0:
Scrapy 2.3.0 (2020-08-04)
-------------------------
Highlights:
* :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` as a storage backend
* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
output items in batches of up to the specified number of items.
It also serves as a workaround for :ref:`delayed file delivery
<delayed-file-delivery>`, which causes Scrapy to only start item delivery
after the crawl has finished when using certain storage backends
(:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
and now :ref:`GCS <topics-feed-storage-gcs>`).
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
allowing usage from outside Scrapy and a separate release schedule
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* Removed the following classes and their parent modules from
``scrapy.linkextractors``:
* ``htmlparser.HtmlParserLinkExtractor``
* ``regex.RegexLinkExtractor``
* ``sgml.BaseSgmlLinkExtractor``
* ``sgml.SgmlLinkExtractor``
Use
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
instead (:issue:`4356`, :issue:`4679`)
Deprecations
~~~~~~~~~~~~
* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
(:issue:`4683`)
New features
~~~~~~~~~~~~
* :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
(:issue:`4250`, :issue:`4434`)
* The :command:`parse` command now allows specifying an output file
(:issue:`4317`, :issue:`4377`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
``--data-raw`` (:issue:`4612`)
* A ``parse`` callback may now be used in built-in spider subclasses, such
as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
:issue:`781`, :issue:`4254` )
Bug fixes
~~~~~~~~~
* Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
:ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
<attrs-items>` (:issue:`4667`, :issue:`4668`)
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
method to ``POST`` when a request body is specified and no request method
is specified (:issue:`4612`)
* The processing of ANSI escape sequences in enabled in Windows 10.0.14393
and later, where it is required for colored output (:issue:`4393`,
:issue:`4403`)
Documentation
~~~~~~~~~~~~~
* Updated the `OpenSSL cipher list format`_ link in the documentation about
the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
* Simplified the code example in :ref:`topics-loaders-dataclass`
(:issue:`4652`)
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
Quality assurance
~~~~~~~~~~~~~~~~~
* The base implementation of :ref:`item loaders <topics-loaders>` has been
moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
:issue:`4516`)
* Fixed a silenced error in some scheduler tests (:issue:`4644`,
:issue:`4645`)
* Renewed the localhost certificate used for SSL tests (:issue:`4650`)
* Removed cookie-handling code specific to Python 2 (:issue:`4682`)
* Stopped using Python 2 unicode literal syntax (:issue:`4704`)
* Stopped using a backlash for line continuation (:issue:`4673`)
* Removed unneeded entries from the MyPy exception list (:issue:`4690`)
* Automated tests now pass on Windows as part of our continuous integration
system (:issue:`4458`)
* Automated tests now pass on the latest PyPy version for supported Python
versions in our continuous integration system (:issue:`4504`)
.. _release-2.2.1:
Scrapy 2.2.1 (2020-07-17)
-------------------------
* The :command:`startproject` command no longer makes unintended changes to
the permissions of files in the destination folder, such as removing
execution permissions (:issue:`4662`, :issue:`4666`)
.. _release-2.2.0:
Scrapy 2.2.0 (2020-06-24)

2
docs/topics/commands.rst
View File

@ -493,6 +493,8 @@ Supported options:
* ``--output`` or ``-o``: dump scraped items to a file
.. versionadded:: 2.3
.. skip: start
Usage example::

6
docs/topics/developer-tools.rst
View File

@ -289,8 +289,10 @@ request::
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments:
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.

8
docs/topics/exporters.rst
View File

@ -166,8 +166,7 @@ BaseItemExporter
By default, this method looks for a serializer :ref:`declared in the item
field <topics-exporters-serializers>` and returns the result of applying
that serializer to the value. If no serializer is found, it returns the
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
value unchanged.
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
@ -217,10 +216,7 @@ BaseItemExporter
.. attribute:: encoding
The encoding that will be used to encode unicode values. This only
affects unicode values (which are always serialized to str using this
encoding). Other value types are passed unchanged to the specific
serialization library.
The output character encoding.
.. attribute:: indent

83
docs/topics/feed-exports.rst
View File

@ -100,6 +100,7 @@ The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
* :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
* :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
mode by default. To use the active connection mode instead, set the
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-s3:
S3
@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting:
* :setting:`FEED_STORAGE_S3_ACL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _topics-feed-storage-gcs:
Google Cloud Storage (GCS)
--------------------------
.. versionadded:: 2.3
The feeds are stored on `Google Cloud Storage`_.
* URI scheme: ``gs``
@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_.
* ``gs://mybucket/path/to/export.csv``
* Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
* Required external libraries: `google-cloud-storage`_.
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following
* :setting:`FEED_STORAGE_GCS_ACL`
* :setting:`GCS_PROJECT_ID`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
.. _topics-feed-storage-stdout:
Standard output
@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
* Required external libraries: none
.. _delayed-file-delivery:
Delayed file delivery
---------------------
As indicated above, some of the described storage backends use delayed file
delivery.
These storage backends do not upload items to the feed URI as those items are
scraped. Instead, Scrapy writes items into a temporary local file, and only
once all the file contents have been written (i.e. at the end of the crawl) is
that file uploaded to the feed URI.
If you want item delivery to start earlier when using one of these storage
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
in multiple files, with the specified maximum item count per file. That way, as
soon as a file reaches the maximum item count, that file is delivered to the
feed URI, allowing item delivery to start way before the end of the crawl.
Settings
========
@ -241,6 +275,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. currentmodule:: scrapy.extensions.feedexport
@ -292,6 +327,7 @@ as a fallback value if that key is not provided for a specific feed definition.
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
.. setting:: FEED_EXPORT_ENCODING
@ -446,6 +482,51 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
FEED_EXPORT_BATCH_ITEM_COUNT
-----------------------------
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
placeholders in the feed URI to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
(e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
alter the number format. For example, to make the batch ID a 5-digit
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
For instance, if your settings include::
FEED_EXPORT_BATCH_ITEM_COUNT = 100
And your :command:`crawl` command line is::
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
The command line above can generate a directory tree like::
->projectname
-->dirname
--->1-filename2020-03-28T14-45-08.237134.json
--->2-filename2020-03-28T14-45-09.148903.json
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
100 items or fewer.
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore

10
docs/topics/loaders.rst
View File

@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example::
default_output_processor = TakeFirst()
name_in = MapCompose(unicode.title)
name_in = MapCompose(str.title)
name_out = Join()
price_in = MapCompose(unicode.strip)
price_in = MapCompose(str.strip)
# ...
@ -237,10 +237,10 @@ metadata. Here is an example::
>>> from scrapy.loader import ItemLoader
>>> il = ItemLoader(item=Product())
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
>>> il.add_value('price', ['&euro;', '<span>1000</span>'])
>>> il.load_item()
{'name': u'Welcome to my website', 'price': u'1000'}
{'name': 'Welcome to my website', 'price': '1000'}
The precedence order, for both input and output processors, is as follows:

30
docs/topics/request-response.rst
View File

@ -51,12 +51,12 @@ Request objects
given, the dict passed in this parameter will be shallow copied.
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
:type body: str or unicode
:param body: the request body. If a string is passed, then it's encoded as
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty bytes object is stored. Regardless of the
type of this argument, the final value stored will be a bytes object
(never a string or ``None``).
:type body: bytes or str
:param headers: the headers of this request. The dict values can be strings
(for single valued headers) or lists (for multi-valued headers). If
@ -106,7 +106,7 @@ Request objects
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to ``str`` (if given as ``unicode``).
body to bytes (if given as a string).
:type encoding: string
:param priority: the priority of this request (defaults to ``0``).
@ -721,7 +721,7 @@ Response objects
.. attribute:: Response.body
The body of this Response. Keep in mind that Response.body
is always a bytes object. If you want the unicode version use
is always a bytes object. If you want the string version use
:attr:`TextResponse.text` (only available in :class:`TextResponse`
and subclasses).
@ -842,9 +842,9 @@ TextResponse objects
is the same as for the :class:`Response` class and is not documented here.
:param encoding: is a string which contains the encoding to use for this
response. If you create a :class:`TextResponse` object with a unicode
response. If you create a :class:`TextResponse` object with a string as
body, it will be encoded using this encoding (remember the body attribute
is always a string). If ``encoding`` is ``None`` (default value), the
is always a bytes object). If ``encoding`` is ``None`` (default value), the
encoding will be looked up in the response headers and body instead.
:type encoding: string
@ -853,7 +853,7 @@ TextResponse objects
.. attribute:: TextResponse.text
Response body, as unicode.
Response body, as a string.
The same as ``response.body.decode(response.encoding)``, but the
result is cached after the first call, so you can access
@ -861,9 +861,11 @@ TextResponse objects
.. note::
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
(typically ``ascii``) instead of the response encoding.
``str(response.body)`` is not a correct way to convert the response
body into a string:
>>> str(b'body')
"b'body'"
.. attribute:: TextResponse.encoding

11
docs/topics/selectors.rst
View File

@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
constructed by passing either :class:`~scrapy.http.TextResponse` object or
markup as an unicode string (in ``text`` argument).
markup as a string (in ``text`` argument).
Usually there is no need to construct Scrapy selectors manually:
``response`` object is available in Spider callbacks, so in most cases
it is more convenient to use ``response.css()`` and ``response.xpath()``
@ -383,7 +384,7 @@ Using selectors with regular expressions
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
data using regular expressions. However, unlike using ``.xpath()`` or
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
``.css()`` methods, ``.re()`` returns a list of strings. So you
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
@ -734,7 +735,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
Example selecting links in list item with a "class" attribute ending with a digit:
>>> from scrapy import Selector
>>> doc = u"""
>>> doc = """
... <div>
... <ul>
... <li class="item-0"><a href="link1.html">first item</a></li>
@ -765,7 +766,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
>>> doc = u"""
>>> doc = """
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel.xpath("//h1")
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
returning a list of strings::
sel.xpath("//h1").getall() # this includes the h1 tag
sel.xpath("//h1/text()").getall() # this excludes the h1 tag

2
docs/utils/linkfix.py
View File

@ -23,7 +23,7 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:

2
extras/qpsclient.py
View File

@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps

1
pytest.ini
View File

@ -40,3 +40,4 @@ flake8-ignore =
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

2
scrapy/VERSION
View File

@ -1 +1 @@
2.2.0
2.3.0

30
scrapy/commands/check.py
View File

@ -78,19 +78,19 @@ class Command(ScrapyCommand):
elif tested_methods:
self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
# start checks
if opts.list:
for spider, methods in sorted(contract_reqs.items()):
if not methods and not opts.verbose:
continue
print(spider)
for method in sorted(methods):
print(' * %s' % method)
else:
start = time.time()
self.crawler_process.start()
stop = time.time()
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())

2
scrapy/commands/view.py
View File

@ -11,7 +11,7 @@ class Command(fetch.Command):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
def add_options(self, parser):
super(Command, self).add_options(parser)
super().add_options(parser)
parser.remove_option("--headers")
def _print_response(self, response, opts):

2
scrapy/contracts/default.py
View File

@ -56,7 +56,7 @@ class ReturnsContract(Contract):
}
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
raise ValueError(

4
scrapy/core/downloader/contextfactory.py
View File

@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),

10
scrapy/core/downloader/handlers/http11.py
View File

@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
return super(TunnelingAgent, self)._requestWithEndpoint(
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
super(ScrapyProxyAgent, self).__init__(
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,

2
scrapy/core/downloader/tls.py
View File

@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
"""
def __init__(self, hostname, ctx, verbose_logging=False):
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
super().__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):

2
scrapy/core/spidermw.py
View File

@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
def _add_middleware(self, mw):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
super()._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):

2
scrapy/crawler.py
View File

@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner):
"""
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)

2
scrapy/downloadermiddlewares/redirect.py
View File

@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = 'METAREFRESH_ENABLED'
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
super().__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')

4
scrapy/exceptions.py
View File

@ -37,7 +37,7 @@ class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason='cancelled'):
super(CloseSpider, self).__init__()
super().__init__()
self.reason = reason
@ -74,7 +74,7 @@ class UsageError(Exception):
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
super().__init__(*a, **kw)
class ScrapyDeprecationWarning(Warning):

10
scrapy/exporters.py
View File

@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
if isinstance(item, dict):
# for dicts try using fields of the first item
self.fields_to_export = list(item.keys())
else:
# use fields declared in Item
self.fields_to_export = list(item.fields.keys())
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
row = list(self._build_row(self.fields_to_export))
self.csv_writer.writerow(row)
@ -305,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
super(PythonItemExporter, self)._configure(options, dont_fail)
super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",

160
scrapy/extensions/feedexport.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
import logging
import os
import re
import sys
import warnings
from datetime import datetime
@ -206,14 +207,16 @@ class FTPFeedStorage(BlockingFeedStorage):
class _FeedSlot:
def __init__(self, file, exporter, storage, uri, format, store_empty):
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
self.file = file
self.exporter = exporter
self.storage = storage
# feed params
self.uri = uri
self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
self.uri_template = uri_template
self.uri = uri
# flags
self.itemcount = 0
self._exporting = False
@ -270,63 +273,112 @@ class FeedExporter:
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
raise NotConfigured
if not self._settings_are_valid():
raise NotConfigured
if not self._exporter_supported(feed['format']):
raise NotConfigured
def open_spider(self, spider):
for uri, feed in self.feeds.items():
uri = uri % self._get_uri_params(spider, feed['uri_params'])
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
format=feed['format'],
fields_to_export=feed['fields'],
encoding=feed['encoding'],
indent=feed['indent'],
)
slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
self.slots.append(slot)
if slot.store_empty:
slot.start_exporting()
uri_params = self._get_uri_params(spider, feed['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
feed=feed,
spider=spider,
uri_template=uri,
))
def close_spider(self, spider):
deferred_list = []
for slot in self.slots:
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
d = defer.maybeDeferred(slot.storage.store, slot.file)
deferred_list.append(d)
continue
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
)
d = self._close_slot(slot, spider)
deferred_list.append(d)
return defer.DeferredList(deferred_list) if deferred_list else None
def _close_slot(self, slot, spider):
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
return defer.maybeDeferred(slot.storage.store, slot.file)
slot.finish_exporting()
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
log_args = {'format': slot.format,
'itemcount': slot.itemcount,
'uri': slot.uri}
d = defer.maybeDeferred(slot.storage.store, slot.file)
# Use `largs=log_args` to copy log_args into function's scope
# instead of using `log_args` from the outer scope
d.addCallback(
lambda _, largs=log_args: logger.info(
logfmt % "Stored", largs, extra={'spider': spider}
)
)
d.addErrback(
lambda f, largs=log_args: logger.error(
logfmt % "Error storing", largs,
exc_info=failure_to_exc_info(f), extra={'spider': spider}
)
)
return d
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
:param spider: user spider
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
"""
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(
file=file,
format=feed['format'],
fields_to_export=feed['fields'],
encoding=feed['encoding'],
indent=feed['indent'],
)
slot = _FeedSlot(
file=file,
exporter=exporter,
storage=storage,
uri=uri,
format=feed['format'],
store_empty=feed['store_empty'],
batch_id=batch_id,
uri_template=uri_template,
)
if slot.store_empty:
slot.start_exporting()
return slot
def item_scraped(self, item, spider):
slots = []
for slot in self.slots:
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
if (
self.feeds[slot.uri_template]['batch_item_count']
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
uri=slot.uri_template % uri_params,
feed=self.feeds[slot.uri_template],
spider=spider,
uri_template=slot.uri_template,
))
else:
slots.append(slot)
self.slots = slots
def _load_components(self, setting_prefix):
conf = without_none_values(self.settings.getwithbase(setting_prefix))
@ -343,6 +395,22 @@ class FeedExporter:
return True
logger.error("Unknown feed format: %(format)s", {'format': format})
def _settings_are_valid(self):
"""
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
"""
for uri_template, values in self.feeds.items():
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
logger.error(
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
''.format(uri_template)
)
return False
return True
def _storage_supported(self, uri):
scheme = urlparse(uri).scheme
if scheme in self.storages:
@ -368,12 +436,14 @@ class FeedExporter:
def _get_storage(self, uri):
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
def _get_uri_params(self, spider, uri_params):
def _get_uri_params(self, spider, uri_params, slot=None):
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
params['time'] = ts
utc_now = datetime.utcnow()
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
params['batch_time'] = utc_now.isoformat().replace(':', '-')
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
uripar_function(params, spider)
return params

8
scrapy/http/headers.py
View File

@ -8,7 +8,7 @@ class Headers(CaselessDict):
def __init__(self, seq=None, encoding='utf-8'):
self.encoding = encoding
super(Headers, self).__init__(seq)
super().__init__(seq)
def normkey(self, key):
"""Normalize key to bytes"""
@ -37,19 +37,19 @@ class Headers(CaselessDict):
def __getitem__(self, key):
try:
return super(Headers, self).__getitem__(key)[-1]
return super().__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
return super(Headers, self).get(key, def_val)[-1]
return super().get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
return super(Headers, self).__getitem__(key)
return super().__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)

8
scrapy/http/request/form.py
View File

@ -24,7 +24,7 @@ class FormRequest(Request):
if formdata and kwargs.get('method') is None:
kwargs['method'] = 'POST'
super(FormRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata
@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
namespaces={
"re": "http://exslt.org/regular-expressions"})
values = [(k, u'' if v is None else v)
values = [(k, '' if v is None else v)
for k, v in (_value(e) for e in inputs)
if k and k not in formdata_keys]
@ -168,7 +168,7 @@ def _select_value(ele, n, v):
# This is a workround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath('.//option[@selected]')
v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
return n, v
@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')

4
scrapy/http/request/json_request.py
View File

@ -32,7 +32,7 @@ class JsonRequest(Request):
if 'method' not in kwargs:
kwargs['method'] = 'POST'
super(JsonRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@ -47,7 +47,7 @@ class JsonRequest(Request):
elif not body_passed and data_passed:
kwargs['body'] = self._dumps(data)
return super(JsonRequest, self).replace(*args, **kwargs)
return super().replace(*args, **kwargs)
def _dumps(self, data):
"""Convert to JSON """

2
scrapy/http/request/rpc.py
View File

@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
if encoding is not None:
kwargs['encoding'] = encoding
super(XmlRpcRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'text/xml')

10
scrapy/http/response/text.py
View File

@ -35,13 +35,13 @@ class TextResponse(Response):
self._cached_benc = None
self._cached_ubody = None
self._cached_selector = None
super(TextResponse, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
super()._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
@ -51,7 +51,7 @@ class TextResponse(Response):
type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
super()._set_body(body)
def replace(self, *args, **kwargs):
kwargs.setdefault('encoding', self.encoding)
@ -166,7 +166,7 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(
return super().follow(
url=url,
callback=callback,
method=method,
@ -226,7 +226,7 @@ class TextResponse(Response):
for sel in selectors:
with suppress(_InvalidSelector):
urls.append(_url_from_selector(sel))
return super(TextResponse, self).follow_all(
return super().follow_all(
urls=urls,
callback=callback,
method=method,

10
scrapy/item.py
View File

@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
class Field(dict):
@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
def __new__(mcs, class_name, bases, attrs):
classcell = attrs.pop('__classcell__', None)
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
_class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
fields = getattr(_class, 'fields', {})
new_attrs = {}
@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
new_attrs['_class'] = _class
if classcell is not None:
new_attrs['__classcell__'] = classcell
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
return super().__new__(mcs, class_name, bases, new_attrs)
class DictItem(MutableMapping, BaseItem):
@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(DictItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
self._values = {}
@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
def __setattr__(self, name, value):
if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
super(DictItem, self).__setattr__(name, value)
super().__setattr__(name, value)
def __len__(self):
return len(self._values)

2
scrapy/linkextractors/__init__.py
View File

@ -65,7 +65,7 @@ class FilteringLinkExtractor:
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(FilteringLinkExtractor, cls).__new__(cls)
return super().__new__(cls)
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):

4
scrapy/linkextractors/lxmlhtml.py
View File

@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
url = safe_url_string(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
link = Link(url, _collect_string_content(el) or '',
nofollow=rel_has_nofollow(el.get('rel')))
links.append(link)
return self._deduplicate_if_needed(links)
@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
strip=strip,
canonicalized=canonicalize
)
super(LxmlLinkExtractor, self).__init__(
super().__init__(
link_extractor=lx,
allow=allow,
deny=deny,

2
scrapy/logformatter.py
View File

@ -44,7 +44,7 @@ class LogFormatter:
def dropped(self, item, exception, response, spider):
return {
'level': logging.INFO, # lowering the level from logging.WARNING
'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
'args': {
'exception': exception,
'item': item,

2
scrapy/pipelines/files.py
View File

@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
)
super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
super().__init__(download_func=download_func, settings=settings)
@classmethod
def from_settings(cls, settings):

3
scrapy/pipelines/images.py
View File

@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
DEFAULT_IMAGES_RESULT_FIELD = 'images'
def __init__(self, store_uri, download_func=None, settings=None):
super(ImagesPipeline, self).__init__(store_uri, settings=settings,
download_func=download_func)
super().__init__(store_uri, settings=settings, download_func=download_func)
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)

10
scrapy/resolver.py
View File

@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
"""
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)
super().__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
# so the input argument above is simply overridden
# to enforce Scrapy's DNS_TIMEOUT setting's value
timeout = (self.timeout,)
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
d = super().getHostByName(name, timeout)
if dnscache.limit:
d.addCallback(self._cache_result, name)
return d
@ -80,16 +80,16 @@ class CachingHostnameResolver:
class CachingResolutionReceiver(resolutionReceiver):
def resolutionBegan(self, resolution):
super(CachingResolutionReceiver, self).resolutionBegan(resolution)
super().resolutionBegan(resolution)
self.resolution = resolution
self.resolved = False
def addressResolved(self, address):
super(CachingResolutionReceiver, self).addressResolved(address)
super().addressResolved(address)
self.resolved = True
def resolutionComplete(self):
super(CachingResolutionReceiver, self).resolutionComplete()
super().resolutionComplete()
if self.resolved:
dnscache[hostName] = self.resolution

2
scrapy/selector/unified.py
View File

@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
kwargs.setdefault('base_url', response.url)
self.response = response
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
super().__init__(text=text, type=st, root=root, **kwargs)

5
scrapy/settings/__init__.py
View File

@ -82,7 +82,8 @@ class BaseSettings(MutableMapping):
def __init__(self, values=None, priority='project'):
self.frozen = False
self.attributes = {}
self.update(values, priority)
if values:
self.update(values, priority)
def __getitem__(self, opt_name):
if opt_name not in self:
@ -439,7 +440,7 @@ class Settings(BaseSettings):
# Do not pass kwarg values here. We don't want to promote user-defined
# dicts, and we want to update, not replace, default dicts with the
# values given by the user
super(Settings, self).__init__()
super().__init__()
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities

1
scrapy/settings/default_settings.py
View File

@ -147,6 +147,7 @@ FEED_STORAGES_BASE = {
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
}
FEED_EXPORT_BATCH_ITEM_COUNT = 0
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',

2
scrapy/spidermiddlewares/httperror.py
View File

@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
def __init__(self, response, *args, **kwargs):
self.response = response
super(HttpError, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
class HttpErrorMiddleware:

4
scrapy/spiders/crawl.py
View File

@ -75,7 +75,7 @@ class CrawlSpider(Spider):
rules = ()
def __init__(self, *a, **kw):
super(CrawlSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
self._compile_rules()
def _parse(self, response, **kwargs):
@ -145,6 +145,6 @@ class CrawlSpider(Spider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
spider = super().from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
return spider

2
scrapy/spiders/init.py
View File

@ -6,7 +6,7 @@ class InitSpider(Spider):
"""Base Spider with initialization facilities"""
def start_requests(self):
self._postinit_reqs = super(InitSpider, self).start_requests()
self._postinit_reqs = super().start_requests()
return iterate_spider_output(self.init_request())
def initialized(self, response=None):

2
scrapy/spiders/sitemap.py
View File

@ -18,7 +18,7 @@ class SitemapSpider(Spider):
sitemap_alternate_links = False
def __init__(self, *a, **kw):
super(SitemapSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, str):

12
scrapy/squeues.py
View File

@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
if not os.path.exists(dirname):
os.makedirs(dirname, exist_ok=True)
super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
super().__init__(path, *args, **kwargs)
return DirectoriesCreated
@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
def push(self, obj):
s = serialize(obj)
super(SerializableQueue, self).push(s)
super().push(s)
def pop(self):
s = super(SerializableQueue, self).pop()
s = super().pop()
if s:
return deserialize(s)
@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
def __init__(self, crawler, key):
self.spider = crawler.spider
super(ScrapyRequestQueue, self).__init__(key)
super().__init__(key)
@classmethod
def from_crawler(cls, crawler, key, *args, **kwargs):
@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
def push(self, request):
request = request_to_dict(request, self.spider)
return super(ScrapyRequestQueue, self).push(request)
return super().push(request)
def pop(self):
request = super(ScrapyRequestQueue, self).pop()
request = super().pop()
if not request:
return None

2
scrapy/statscollectors.py
View File

@ -54,7 +54,7 @@ class StatsCollector:
class MemoryStatsCollector(StatsCollector):
def __init__(self, crawler):
super(MemoryStatsCollector, self).__init__(crawler)
super().__init__(crawler)
self.spider_stats = {}
def _persist_stats(self, stats, spider):

1
scrapy/utils/conf.py
View File

@ -115,6 +115,7 @@ def get_sources(use_closest=True):
def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))

3
scrapy/utils/curl.py
View File

@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
:param str curl_command: string containing the curl command
:param bool ignore_unknown_options: If true, only a warning is emitted when
cURL options are unknown. Otherwise raises an error. (default: True)
cURL options are unknown. Otherwise
raises an error. (default: True)
:return: dictionary of Request kwargs
"""

14
scrapy/utils/datatypes.py
View File

@ -15,7 +15,7 @@ class CaselessDict(dict):
__slots__ = ()
def __init__(self, seq=None):
super(CaselessDict, self).__init__()
super().__init__()
if seq:
self.update(seq)
@ -53,7 +53,7 @@ class CaselessDict(dict):
def update(self, seq):
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super(CaselessDict, self).update(iseq)
super().update(iseq)
@classmethod
def fromkeys(cls, keys, value=None):
@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
"""
def __init__(self, limit=None):
super(LocalCache, self).__init__()
super().__init__()
self.limit = limit
def __setitem__(self, key, value):
if self.limit:
while len(self) >= self.limit:
self.popitem(last=False)
super(LocalCache, self).__setitem__(key, value)
super().__setitem__(key, value)
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
"""
def __init__(self, limit=None):
super(LocalWeakReferencedCache, self).__init__()
super().__init__()
self.data = LocalCache(limit=limit)
def __setitem__(self, key, value):
try:
super(LocalWeakReferencedCache, self).__setitem__(key, value)
super().__setitem__(key, value)
except TypeError:
pass # key is not weak-referenceable, skip caching
def __getitem__(self, key):
try:
return super(LocalWeakReferencedCache, self).__getitem__(key)
return super().__getitem__(key)
except (TypeError, KeyError):
return None # key is either not weak-referenceable or not cached

8
scrapy/utils/deprecate.py
View File

@ -57,7 +57,7 @@ def create_deprecated_class(
warned_on_subclass = False
def __new__(metacls, name, bases, clsdict_):
cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
cls = super().__new__(metacls, name, bases, clsdict_)
if metacls.deprecated_class is None:
metacls.deprecated_class = cls
return cls
@ -73,7 +73,7 @@ def create_deprecated_class(
if warn_once:
msg += ' (warning only on first subclass, there may be others)'
warnings.warn(msg, warn_category, stacklevel=2)
super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
super().__init__(name, bases, clsdict_)
# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
@ -88,7 +88,7 @@ def create_deprecated_class(
# is the deprecated class itself - subclasses of the
# deprecated class should not use custom `__subclasscheck__`
# method.
return super(DeprecatedClass, cls).__subclasscheck__(sub)
return super().__subclasscheck__(sub)
if not inspect.isclass(sub):
raise TypeError("issubclass() arg 1 must be a class")
@ -102,7 +102,7 @@ def create_deprecated_class(
msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
new=_clspath(new_class, new_class_path))
warnings.warn(msg, warn_category, stacklevel=2)
return super(DeprecatedClass, cls).__call__(*args, **kwargs)
return super().__call__(*args, **kwargs)
deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})

2
scrapy/utils/log.py
View File

@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
"""Record log levels count into a crawler stats"""
def __init__(self, crawler, *args, **kwargs):
super(LogCounterHandler, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.crawler = crawler
def emit(self, record):

5
scrapy/utils/python.py
View File

@ -6,10 +6,12 @@ import gc
import inspect
import re
import sys
import warnings
import weakref
from functools import partial, wraps
from itertools import chain
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.decorators import deprecated
@ -127,6 +129,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
the start position of the match, and the ending (regarding the entire text).
"""
def _chunk_iter():
offset = len(text)
while True:
@ -158,6 +161,7 @@ def memoizemethod_noargs(method):
if self not in cache:
cache[self] = method(self, *args, **kwargs)
return cache[self]
return new_method
@ -276,6 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
class WeakKeyCache:
def __init__(self, default_factory):
warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
self.default_factory = default_factory
self._weakdict = weakref.WeakKeyDictionary()

2
scrapy/utils/serialize.py
View File

@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
elif isinstance(o, Response):
return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
else:
return super(ScrapyJSONEncoder, self).default(o)
return super().default(o)
class ScrapyJSONDecoder(json.JSONDecoder):

4
scrapy/utils/testsite.py
View File

@ -7,12 +7,12 @@ class SiteTest:
def setUp(self):
from twisted.internet import reactor
super(SiteTest, self).setUp()
super().setUp()
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
self.baseurl = "http://localhost:%d/" % self.site.getHost().port
def tearDown(self):
super(SiteTest, self).tearDown()
super().tearDown()
self.site.stopListening()
def url(self, path):

1
setup.py
View File

@ -23,7 +23,6 @@ install_requires = [
'cryptography>=2.0',
'cssselect>=0.9.1',
'itemloaders>=1.0.1',
'lxml>=3.5.0',
'parsel>=1.5.0',
'PyDispatcher>=2.0.5',
'pyOpenSSL>=16.2.0',

16
tests/spiders.py
View File

@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue
class MockServerSpider(Spider):
def __init__(self, mockserver=None, *args, **kwargs):
super(MockServerSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.mockserver = mockserver
@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):
name = 'meta'
def __init__(self, *args, **kwargs):
super(MetaSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.meta = {}
def closed(self, reason):
@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):
link_extractor = LinkExtractor()
def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
super(FollowAllSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.urls_visited = []
self.times = []
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):
name = 'delay'
def __init__(self, n=1, b=0, *args, **kwargs):
super(DelaySpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.n = n
self.b = b
self.t1 = self.t2 = self.t2_err = 0
@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):
name = 'simple'
def __init__(self, url="http://localhost:8998", *args, **kwargs):
super(SimpleSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.start_urls = [url]
def parse(self, response):
@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):
name = 'item'
def parse(self, response):
for request in super(ItemSpider, self).parse(response):
for request in super().parse(response):
yield request
yield Item()
yield {}
@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):
raise self.exception_cls('Expected exception')
def parse(self, response):
for request in super(ErrorSpider, self).parse(response):
for request in super().parse(response):
yield request
self.raise_exception()
@ -239,7 +239,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
yield Request(url, dont_filter=self.dont_filter)
def __init__(self, url="http://localhost:8998", *args, **kwargs):
super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.visited = 0
def parse(self, response):

2
tests/test_cmdline/__init__.py
View File

@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
'EXTENSIONS=' + json.dumps(EXTENSIONS))
# XXX: There's gotta be a smarter way to do this...
self.assertNotIn("...", settingsstr)
for char in ("'", "<", ">", 'u"'):
for char in ("'", "<", ">"):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())

97
tests/test_command_check.py Normal file
View File

@ -0,0 +1,97 @@
from os.path import join, abspath
from tests.test_commands import CommandTest
class CheckCommandTest(CommandTest):
command = 'check'
def setUp(self):
super(CheckCommandTest, self).setUp()
self.spider_name = 'check_spider'
self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
def _write_contract(self, contracts, parse_def):
with open(self.spider, 'w') as file:
file.write("""
import scrapy
class CheckSpider(scrapy.Spider):
name = '{0}'
start_urls = ['http://example.com']
def parse(self, response, **cb_kwargs):
\"\"\"
@url http://example.com
{1}
\"\"\"
{2}
""".format(self.spider_name, contracts, parse_def))
def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def)
p, out, err = self.proc('check')
self.assertNotIn('F', out)
self.assertIn('OK', err)
self.assertEqual(p.returncode, 0)
def test_check_returns_requests_contract(self):
contracts = """
@returns requests 1
"""
parse_def = """
yield scrapy.Request(url='http://next-url.com')
"""
self._test_contract(contracts, parse_def)
def test_check_returns_items_contract(self):
contracts = """
@returns items 1
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
"""
self._test_contract(contracts, parse_def)
def test_check_cb_kwargs_contract(self):
contracts = """
@cb_kwargs {"arg1": "val1", "arg2": "val2"}
"""
parse_def = """
if len(cb_kwargs.items()) == 0:
raise Exception("Callback args not set")
"""
self._test_contract(contracts, parse_def)
def test_check_scrapes_contract(self):
contracts = """
@scrapes key1 key2
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
"""
self._test_contract(contracts, parse_def)
def test_check_all_default_contracts(self):
contracts = """
@returns items 1
@returns requests 1
@scrapes key1 key2
@cb_kwargs {"arg1": "val1", "arg2": "val2"}
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
yield scrapy.Request(url='http://next-url.com')
if len(cb_kwargs.items()) == 0:
raise Exception("Callback args not set")
"""
self._test_contract(contracts, parse_def)
def test_SCRAPY_CHECK_set(self):
parse_def = """
import os
if not os.environ.get('SCRAPY_CHECK'):
raise Exception('SCRAPY_CHECK not set')
"""
self._test_contract(parse_def=parse_def)

2
tests/test_command_parse.py
View File

@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'
def setUp(self):
super(ParseCommandTest, self).setUp()
super().setUp()
self.spider_name = 'parse_spider'
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
with open(fname, 'w') as f:

4
tests/test_commands.py
View File

@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
class StartprojectTemplatesTest(ProjectTest):
def setUp(self):
super(StartprojectTemplatesTest, self).setUp()
super().setUp()
self.tmpl = join(self.temp_path, 'templates')
self.tmpl_proj = join(self.tmpl, 'project')
@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):
class CommandTest(ProjectTest):
def setUp(self):
super(CommandTest, self).setUp()
super().setUp()
self.call('startproject', self.project_name)
self.cwd = join(self.temp_path, self.project_name)
self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name

2
tests/test_contracts.py
View File

@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):
name = 'test_same_url'
def __init__(self, *args, **kwargs):
super(TestSameUrlSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.visited = 0
def start_requests(s):

10
tests/test_downloader_handlers.py
View File

@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super(Https11InvalidDNSId, self).setUp()
super().setUp()
self.host = '127.0.0.1'
@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
)
super(Https11InvalidDNSPattern, self).setUp()
super().setUp()
class Https11CustomCiphers(unittest.TestCase):
@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
def test_default_mediatype(self):
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "iso-8859-7")
@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
def test_text_charset(self):
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(response.body, b'\xbe\xd3\xbe')
self.assertEqual(response.encoding, "iso-8859-7")
@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
def test_mediatype_parameters(self):
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "utf-8")

16
tests/test_downloadermiddleware_cookies.py
View File

@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
# 3) Unicode string
req3 = Request('http://example.org', cookies={'a': u'á'})
# 3) String
req3 = Request('http://example.org', cookies={'a': 'á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes
req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
# 3) Unicode string
req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
# 3) String
req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')

4
tests/test_downloadermiddleware_httpcache.py
View File

@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
def _get_settings(self, **new_settings):
new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
return super()._get_settings(**new_settings)
def test_custom_dbm_module_loaded(self):
# make sure our dbm module has been loaded
@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
def _get_settings(self, **new_settings):
new_settings.setdefault('HTTPCACHE_GZIP', True)
return super(FilesystemStorageTest, self)._get_settings(**new_settings)
return super()._get_settings(**new_settings)
class DummyPolicyTest(_BaseTest):

6
tests/test_downloadermiddleware_httpproxy.py
View File

@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
def test_proxy_auth_encoding(self):
# utf-8 encoding
os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
mw = HttpProxyMiddleware(auth_encoding='utf-8')
req = Request('http://scrapytest.org')
assert mw.process_request(req, spider) is None
@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
# proxy from request.meta
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
# proxy from request.meta, latin-1 encoding
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')

6
tests/test_downloadermiddleware_redirect.py
View File

@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def test_latin1_location(self):
req = Request('http://scrapytest.org/first')
latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1
latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1
resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def test_utf8_location(self):
req = Request('http://scrapytest.org/first')
utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding
utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding
resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
self.mw = MetaRefreshMiddleware.from_crawler(crawler)
def _body(self, interval=5, url='http://example.org/newpage'):
html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
return html.format(interval, url).encode('utf-8')
def test_priority_adjust(self):

8
tests/test_downloadermiddleware_robotstxt.py
View File

@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def _get_successful_crawler(self):
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
ROBOTS = u"""
ROBOTS = """
User-Agent: *
Disallow: /admin/
Disallow: /static/
@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
self.assertIgnored(Request('http://site.local/admin/main'), middleware),
self.assertIgnored(Request('http://site.local/static/'), middleware),
self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
], fireOnOneErrback=True)
def test_robotstxt_ready_parser(self):
@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
skip = "Rerp parser is not installed"
def setUp(self):
super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
super().setUp()
self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
skip = "Reppy parser is not installed"
def setUp(self):
super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
super().setUp()
self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')

241
tests/test_exporters.py
View File

@ -8,6 +8,7 @@ from io import BytesIO
from datetime import datetime
import lxml.etree
from itemadapter import ItemAdapter
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
@ -23,10 +24,37 @@ class TestItem(Item):
age = Field()
def custom_serializer(value):
return str(int(value) + 2)
class CustomFieldItem(Item):
name = Field()
age = Field(serializer=custom_serializer)
try:
from dataclasses import make_dataclass, field
except ImportError:
TestDataClass = None
CustomFieldDataclass = None
else:
TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
CustomFieldDataclass = make_dataclass(
"CustomFieldDataclass",
[("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
)
class BaseItemExporterTest(unittest.TestCase):
item_class = TestItem
custom_field_item_class = CustomFieldItem
def setUp(self):
self.i = TestItem(name=u'John\xa3', age=u'22')
if self.item_class is None:
raise unittest.SkipTest("item class is None")
self.i = self.item_class(name='John\xa3', age='22')
self.output = BytesIO()
self.ie = self._get_exporter()
@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase):
def _assert_expected_item(self, exported_dict):
for k, v in exported_dict.items():
exported_dict[k] = to_unicode(v)
self.assertEqual(self.i, exported_dict)
self.assertEqual(self.i, self.item_class(**exported_dict))
def _get_nonstring_types_item(self):
return {
@ -63,37 +91,36 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertItemExportWorks(self.i)
def test_export_dict_item(self):
self.assertItemExportWorks(dict(self.i))
self.assertItemExportWorks(ItemAdapter(self.i).asdict())
def test_serialize_field(self):
res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name'])
self.assertEqual(res, u'John\xa3')
a = ItemAdapter(self.i)
res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
self.assertEqual(res, 'John\xa3')
res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age'])
self.assertEqual(res, u'22')
res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
self.assertEqual(res, '22')
def test_fields_to_export(self):
ie = self._get_exporter(fields_to_export=['name'])
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
_, name = list(ie._get_serialized_fields(self.i))[0]
assert isinstance(name, str)
self.assertEqual(name, u'John\xa3')
self.assertEqual(name, 'John\xa3')
def test_field_custom_serializer(self):
def custom_serializer(value):
return str(int(value) + 2)
class CustomFieldItem(Item):
name = Field()
age = Field(serializer=custom_serializer)
i = CustomFieldItem(name=u'John\xa3', age=u'22')
i = self.custom_field_item_class(name='John\xa3', age='22')
a = ItemAdapter(i)
ie = self._get_exporter()
self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
class BaseItemExporterDataclassTest(BaseItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class PythonItemExporterTest(BaseItemExporterTest):
@ -105,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
PythonItemExporter(invalid_option='something')
def test_nested_item(self):
i1 = TestItem(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=i1)
i3 = TestItem(name=u'Jesus', age=i2)
i1 = self.item_class(name='Joseph', age='22')
i2 = dict(name='Maria', age=i1)
i3 = self.item_class(name='Jesus', age=i2)
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(type(exported), dict)
self.assertEqual(
exported,
{'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
{'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
)
self.assertEqual(type(exported['age']), dict)
self.assertEqual(type(exported['age']['age']), dict)
def test_export_list(self):
i1 = TestItem(name=u'Joseph', age='22')
i2 = TestItem(name=u'Maria', age=[i1])
i3 = TestItem(name=u'Jesus', age=[i2])
i1 = self.item_class(name='Joseph', age='22')
i2 = self.item_class(name='Maria', age=[i1])
i3 = self.item_class(name='Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
exported,
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
{'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
)
self.assertEqual(type(exported['age'][0]), dict)
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_item_dict_list(self):
i1 = TestItem(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=[i1])
i3 = TestItem(name=u'Jesus', age=[i2])
i1 = self.item_class(name='Joseph', age='22')
i2 = dict(name='Maria', age=[i1])
i3 = self.item_class(name='Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
exported,
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
{'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
)
self.assertEqual(type(exported['age'][0]), dict)
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_binary(self):
exporter = PythonItemExporter(binary=True)
value = TestItem(name=u'John\xa3', age=u'22')
value = self.item_class(name='John\xa3', age='22')
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
self.assertEqual(expected, exporter.export_item(value))
@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(exported, item)
class PythonItemExporterDataclassTest(PythonItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class PprintItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest):
self._assert_expected_item(eval(self.output.getvalue()))
class PprintItemExporterDataclassTest(PprintItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class PickleItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
self._assert_expected_item(pickle.loads(self.output.getvalue()))
def test_export_multiple_items(self):
i1 = TestItem(name='hello', age='world')
i2 = TestItem(name='bye', age='world')
i1 = self.item_class(name='hello', age='world')
i2 = self.item_class(name='bye', age='world')
f = BytesIO()
ie = PickleItemExporter(f)
ie.start_exporting()
@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
ie.export_item(i2)
ie.finish_exporting()
f.seek(0)
self.assertEqual(pickle.load(f), i1)
self.assertEqual(pickle.load(f), i2)
self.assertEqual(self.item_class(**pickle.load(f)), i1)
self.assertEqual(self.item_class(**pickle.load(f)), i2)
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest):
self.assertEqual(pickle.loads(fp.getvalue()), item)
class PickleItemExporterDataclassTest(PickleItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class MarshalItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest):
self.assertEqual(marshal.load(fp), item)
class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
@ -232,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest):
def test_header_export_all(self):
self.assertExportResult(
item=self.i,
fields_to_export=self.i.fields.keys(),
fields_to_export=ItemAdapter(self.i).field_names(),
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
)
def test_header_export_all_dict(self):
self.assertExportResult(
item=dict(self.i),
item=ItemAdapter(self.i).asdict(),
expected=b'age,name\r\n22,John\xc2\xa3\r\n',
)
def test_header_export_single_field(self):
for item in [self.i, dict(self.i)]:
for item in [self.i, ItemAdapter(self.i).asdict()]:
self.assertExportResult(
item=item,
fields_to_export=['age'],
@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
def test_header_export_two_items(self):
for item in [self.i, dict(self.i)]:
for item in [self.i, ItemAdapter(self.i).asdict()]:
output = BytesIO()
ie = CsvItemExporter(output)
ie.start_exporting()
@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
def test_header_no_header_line(self):
for item in [self.i, dict(self.i)]:
for item in [self.i, ItemAdapter(self.i).asdict()]:
self.assertExportResult(
item=item,
include_headers_line=False,
@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest):
)
class CsvItemExporterDataclassTest(CsvItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class XmlItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
def xmltuple(elem):
children = list(elem.iterchildren())
if children:
return [(child.tag, sorted(xmltuple(child)))
for child in children]
return [(child.tag, sorted(xmltuple(child))) for child in children]
else:
return [(elem.tag, [(elem.text, ())])]
@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest):
def test_multivalued_fields(self):
self.assertExportResult(
TestItem(name=[u'John\xa3', u'Doe']),
(
b'<?xml version="1.0" encoding="utf-8"?>\n'
b'<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
)
self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
b"""<?xml version="1.0" encoding="utf-8"?>\n
<items>
<item>
<name><value>John\xc2\xa3</value><value>Doe</value></name>
<age><value>1</value><value>2</value><value>3</value></age>
</item>
</items>
"""
)
def test_nested_item(self):
i1 = TestItem(name=u'foo\xa3hoo', age='22')
i2 = dict(name=u'bar', age=i1)
i3 = TestItem(name=u'buz', age=i2)
i1 = dict(name='foo\xa3hoo', age='22')
i2 = dict(name='bar', age=i1)
i3 = self.item_class(name='buz', age=i2)
self.assertExportResult(
i3,
@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_list_item(self):
i1 = TestItem(name=u'foo')
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
i3 = TestItem(name=u'buz', age=[i1, i2])
i1 = dict(name='foo')
i2 = dict(name='bar', v2={"egg": ["spam"]})
i3 = self.item_class(name='buz', age=[i1, i2])
self.assertExportResult(
i3,
@ -412,21 +467,27 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
class XmlItemExporterDataclassTest(XmlItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class JsonLinesItemExporterTest(BaseItemExporterTest):
_expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
_expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
def _get_exporter(self, **kwargs):
return JsonLinesItemExporter(self.output, **kwargs)
def _check_output(self):
exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, dict(self.i))
self.assertEqual(exported, ItemAdapter(self.i).asdict())
def test_nested_item(self):
i1 = TestItem(name=u'Joseph', age='22')
i2 = dict(name=u'Maria', age=i1)
i3 = TestItem(name=u'Jesus', age=i2)
i1 = self.item_class(name='Joseph', age='22')
i2 = dict(name='Maria', age=i1)
i3 = self.item_class(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.assertEqual(exported, item)
class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class JsonItemExporterTest(JsonLinesItemExporterTest):
_expected_nested = [JsonLinesItemExporterTest._expected_nested]
@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
def _check_output(self):
exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, [dict(self.i)])
self.assertEqual(exported, [ItemAdapter(self.i).asdict()])
def assertTwoItemsExported(self, item):
self.ie.start_exporting()
@ -466,34 +533,34 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.export_item(item)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, [dict(item), dict(item)])
self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()])
def test_two_items(self):
self.assertTwoItemsExported(self.i)
def test_two_dict_items(self):
self.assertTwoItemsExported(dict(self.i))
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
def test_nested_item(self):
i1 = TestItem(name=u'Joseph\xa3', age='22')
i2 = TestItem(name=u'Maria', age=i1)
i3 = TestItem(name=u'Jesus', age=i2)
i1 = self.item_class(name='Joseph\xa3', age='22')
i2 = self.item_class(name='Maria', age=i1)
i3 = self.item_class(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
self.assertEqual(exported, [expected])
def test_nested_dict_item(self):
i1 = dict(name=u'Joseph\xa3', age='22')
i2 = TestItem(name=u'Maria', age=i1)
i3 = dict(name=u'Jesus', age=i2)
i1 = dict(name='Joseph\xa3', age='22')
i2 = self.item_class(name='Maria', age=i1)
i3 = dict(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
self.assertEqual(exported, [expected])
def test_nonstring_types_item(self):
@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertEqual(exported, [item])
class CustomItemExporterTest(unittest.TestCase):
class JsonItemExporterDataclassTest(JsonItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class CustomExporterItemTest(unittest.TestCase):
item_class = TestItem
def setUp(self):
if self.item_class is None:
raise unittest.SkipTest("item class is None")
def test_exporter_custom_serializer(self):
class CustomItemExporter(BaseItemExporter):
@ -514,18 +593,24 @@ class CustomItemExporterTest(unittest.TestCase):
if name == 'age':
return str(int(value) + 1)
else:
return super(CustomItemExporter, self).serialize_field(field, name, value)
return super().serialize_field(field, name, value)
i = TestItem(name=u'John', age='22')
i = self.item_class(name='John', age='22')
a = ItemAdapter(i)
ie = CustomItemExporter()
self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John')
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23')
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
i2 = {'name': u'John', 'age': '22'}
i2 = {'name': 'John', 'age': '22'}
self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
class CustomExporterDataclassTest(CustomExporterItemTest):
item_class = TestDataClass
if __name__ == '__main__':
unittest.main()

508
tests/test_feedexport.py
View File

@ -6,6 +6,8 @@ import shutil
import string
import tempfile
import warnings
from abc import ABC, abstractmethod
from collections import defaultdict
from io import BytesIO
from logging import getLogger
from pathlib import Path
@ -24,9 +26,11 @@ from zope.interface.verify import verifyObject
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import NotConfigured
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
BlockingFeedStorage,
FeedExporter,
FileFeedStorage,
FTPFeedStorage,
GCSFeedStorage,
@ -91,6 +95,7 @@ class FTPFeedStorageTest(unittest.TestCase):
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
name = 'test_spider'
crawler = get_crawler(settings_dict=settings)
spider = TestSpider.from_crawler(crawler)
return spider
@ -144,6 +149,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
name = 'test_spider'
crawler = get_crawler(settings_dict=settings)
spider = TestSpider.from_crawler(crawler)
return spider
@ -502,23 +508,84 @@ class LogOnStoreFileStorage:
file.close()
class FeedExportTest(unittest.TestCase):
class FeedExportTestBase(ABC, unittest.TestCase):
__test__ = False
class MyItem(scrapy.Item):
foo = scrapy.Field()
egg = scrapy.Field()
baz = scrapy.Field()
def _random_temp_filename(self, inter_dir=''):
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
return os.path.join(self.temp_dir, inter_dir, filename)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
def _random_temp_filename(self):
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
return os.path.join(self.temp_dir, filename)
@defer.inlineCallbacks
def exported_data(self, items, settings):
"""
Return exported data which a spider yielding ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = 'testspider'
def parse(self, response):
for item in items:
yield item
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
def exported_no_data(self, settings):
"""
Return exported data which a spider yielding no ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = 'testspider'
def parse(self, response):
pass
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
def assertExported(self, items, header, rows, settings=None, ordered=True):
yield self.assertExportedCsv(items, header, rows, settings, ordered)
yield self.assertExportedJsonLines(items, rows, settings)
yield self.assertExportedXml(items, rows, settings)
yield self.assertExportedPickle(items, rows, settings)
yield self.assertExportedMarshal(items, rows, settings)
yield self.assertExportedMultiple(items, rows, settings)
@abstractmethod
def run_and_export(self, spider_cls, settings):
pass
def _load_until_eof(self, data, load_func):
result = []
with tempfile.TemporaryFile() as temp:
temp.write(data)
temp.seek(0)
while True:
try:
result.append(load_func(temp))
except EOFError:
break
return result
class FeedExportTest(FeedExportTestBase):
__test__ = True
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings):
@ -559,35 +626,6 @@ class FeedExportTest(unittest.TestCase):
return content
@defer.inlineCallbacks
def exported_data(self, items, settings):
"""
Return exported data which a spider yielding ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = 'testspider'
def parse(self, response):
for item in items:
yield item
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
def exported_no_data(self, settings):
"""
Return exported data which a spider yielding no ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = 'testspider'
def parse(self, response):
pass
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
settings = settings or {}
@ -653,18 +691,6 @@ class FeedExportTest(unittest.TestCase):
json_rows = json.loads(to_unicode(data['json']))
self.assertEqual(rows, json_rows)
def _load_until_eof(self, data, load_func):
result = []
with tempfile.TemporaryFile() as temp:
temp.write(data)
temp.seek(0)
while True:
try:
result.append(load_func(temp))
except EOFError:
break
return result
@defer.inlineCallbacks
def assertExportedPickle(self, items, rows, settings=None):
settings = settings or {}
@ -693,15 +719,6 @@ class FeedExportTest(unittest.TestCase):
result = self._load_until_eof(data['marshal'], load_func=marshal.load)
self.assertEqual(expected, result)
@defer.inlineCallbacks
def assertExported(self, items, header, rows, settings=None, ordered=True):
yield self.assertExportedCsv(items, header, rows, settings, ordered)
yield self.assertExportedJsonLines(items, rows, settings)
yield self.assertExportedXml(items, rows, settings)
yield self.assertExportedPickle(items, rows, settings)
yield self.assertExportedMarshal(items, rows, settings)
yield self.assertExportedMultiple(items, rows, settings)
@defer.inlineCallbacks
def test_export_items(self):
# feed exporters use field names from Item
@ -725,7 +742,7 @@ class FeedExportTest(unittest.TestCase):
},
}
data = yield self.exported_no_data(settings)
self.assertEqual(data[fmt], b'')
self.assertEqual(b'', data[fmt])
@defer.inlineCallbacks
def test_export_no_items_store_empty(self):
@ -745,7 +762,7 @@ class FeedExportTest(unittest.TestCase):
'FEED_EXPORT_INDENT': None,
}
data = yield self.exported_no_data(settings)
self.assertEqual(data[fmt], expctd)
self.assertEqual(expctd, data[fmt])
@defer.inlineCallbacks
def test_export_no_items_multiple_feeds(self):
@ -857,7 +874,7 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_encoding(self):
items = [dict({'foo': u'Test\xd6'})]
items = [dict({'foo': 'Test\xd6'})]
formats = {
'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
@ -902,7 +919,7 @@ class FeedExportTest(unittest.TestCase):
@defer.inlineCallbacks
def test_export_multiple_configs(self):
items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
formats = {
'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@ -1163,3 +1180,376 @@ class FeedExportTest(unittest.TestCase):
print(log)
for fmt in ['json', 'xml', 'csv']:
self.assertIn('Error storing %s feed (2 items)' % fmt, str(log))
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True
_file_mark = '_%(batch_time)s_#%(batch_id)02d_'
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings):
""" Run spider with specified settings; return exported data. """
def build_url(path):
if path[0] != '/':
path = '/' + path
return urljoin('file:', path)
FEEDS = settings.get('FEEDS') or {}
settings['FEEDS'] = {
build_url(file_path): feed
for file_path, feed in FEEDS.items()
}
content = defaultdict(list)
try:
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
for path, feed in FEEDS.items():
dir_name = os.path.dirname(path)
for file in sorted(os.listdir(dir_name)):
with open(os.path.join(dir_name, file), 'rb') as f:
data = f.read()
content[feed['format']].append(data)
finally:
self.tearDown()
defer.returnValue(content)
@defer.inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['jl']:
got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()]
expected_batch, rows = rows[:batch_size], rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
data = yield self.exported_data(items, settings)
for batch in data['csv']:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
self.assertEqual(list(header), got_batch.fieldnames)
expected_batch, rows = rows[:batch_size], rows[batch_size:]
self.assertEqual(expected_batch, list(got_batch))
@defer.inlineCallbacks
def assertExportedXml(self, items, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['xml']:
root = lxml.etree.fromstring(batch)
got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
expected_batch, rows = rows[:batch_size], rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def assertExportedMultiple(self, items, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
# XML
xml_rows = rows.copy()
for batch in data['xml']:
root = lxml.etree.fromstring(batch)
got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')]
expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
# JSON
json_rows = rows.copy()
for batch in data['json']:
got_batch = json.loads(batch.decode('utf-8'))
expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def assertExportedPickle(self, items, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import pickle
for batch in data['pickle']:
got_batch = self._load_until_eof(batch, load_func=pickle.load)
expected_batch, rows = rows[:batch_size], rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def assertExportedMarshal(self, items, rows, settings=None):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import marshal
for batch in data['marshal']:
got_batch = self._load_until_eof(batch, load_func=marshal.load)
expected_batch, rows = rows[:batch_size], rows[batch_size:]
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def test_export_items(self):
""" Test partial deliveries in all supported formats """
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
]
rows = [
{'egg': 'spam1', 'foo': 'bar1', 'baz': ''},
{'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'},
{'foo': 'bar3', 'baz': 'quux3', 'egg': ''}
]
settings = {
'FEED_EXPORT_BATCH_ITEM_COUNT': 2
}
header = self.MyItem.fields.keys()
yield self.assertExported(items, header, rows, settings=Settings(settings))
def test_wrong_path(self):
""" If path is without %(batch_time)s and %(batch_id) an exception must be raised """
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'xml'},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
crawler = get_crawler(settings_dict=settings)
self.assertRaises(NotConfigured, FeedExporter, crawler)
@defer.inlineCallbacks
def test_export_no_items_not_store_empty(self):
for fmt in ('json', 'jsonlines', 'xml', 'csv'):
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
data = yield self.exported_no_data(settings)
data = dict(data)
self.assertEqual(b'', data[fmt][0])
@defer.inlineCallbacks
def test_export_no_items_store_empty(self):
formats = (
('json', b'[]'),
('jsonlines', b''),
('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
('csv', b''),
)
for fmt, expctd in formats:
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
},
'FEED_STORE_EMPTY': True,
'FEED_EXPORT_INDENT': None,
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_no_data(settings)
data = dict(data)
self.assertEqual(expctd, data[fmt][0])
@defer.inlineCallbacks
def test_export_multiple_configs(self):
items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
formats = {
'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
'[\n{"bar": "BAR1"}\n]'.encode('utf-8')],
'xml': [
(
'<?xml version="1.0" encoding="latin-1"?>\n'
'<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'
).encode('latin-1'),
(
'<?xml version="1.0" encoding="latin-1"?>\n'
'<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'
).encode('latin-1')
],
'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
}
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
'format': 'json',
'indent': 0,
'fields': ['bar'],
'encoding': 'utf-8',
},
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
'format': 'xml',
'indent': 2,
'fields': ['foo'],
'encoding': 'latin-1',
},
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
'format': 'csv',
'indent': None,
'fields': ['foo', 'bar'],
'encoding': 'utf-8',
},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt]):
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def test_batch_item_count_feeds_setting(self):
items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
formats = {
'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
'[{"foo": "FOO1"}]'.encode('utf-8')],
}
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
'format': 'json',
'indent': None,
'encoding': 'utf-8',
'batch_item_count': 1,
},
},
}
data = yield self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt]):
self.assertEqual(expected_batch, got_batch)
@defer.inlineCallbacks
def test_batch_path_differ(self):
"""
Test that the name of all batch files differ from each other.
So %(batch_time)s replaced with the current date.
"""
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
]
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
'format': 'json',
},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
}
data = yield self.exported_data(items, settings)
self.assertEqual(len(items) + 1, len(data['json']))
@defer.inlineCallbacks
def test_s3_export(self):
"""
Test export of items into s3 bucket.
S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
to perform this test:
[testenv]
setenv =
AWS_SECRET_ACCESS_KEY = ABCD
AWS_ACCESS_KEY_ID = EFGH
S3_TEST_BUCKET_NAME = IJKL
"""
try:
import boto3
except ImportError:
raise unittest.SkipTest("S3FeedStorage requires boto3")
assert_aws_environ()
s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if not s3_test_bucket_name:
raise unittest.SkipTest("No S3 BUCKET available for testing")
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
prefix = 'tmp/{filename}'.format(filename=filename)
s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(
bucket_name=s3_test_bucket_name, prefix=prefix
)
storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
settings = Settings({
'FEEDS': {
s3_test_file_uri: {
'format': 'json',
},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
})
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
]
verifyObject(IFeedStorage, storage)
class TestSpider(scrapy.Spider):
name = 'testspider'
def parse(self, response):
for item in items:
yield item
s3 = boto3.resource('s3')
my_bucket = s3.Bucket(s3_test_bucket_name)
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))
TestSpider.start_urls = [s.url('/')]
yield runner.crawl(TestSpider)
for file_uri in my_bucket.objects.filter(Prefix=prefix):
content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
if not content and not items:
break
content = json.loads(content.decode('utf-8'))
expected_batch, items = items[:batch_size], items[batch_size:]
self.assertEqual(expected_batch, content)

6
tests/test_http_headers.py
View File

@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
assert h.getlist('X-Forwarded-For') is not hlist
def test_encode_utf8(self):
h = Headers({u'key': u'\xa3'}, encoding='utf-8')
h = Headers({'key': '\xa3'}, encoding='utf-8')
key, val = dict(h).popitem()
assert isinstance(key, bytes), key
assert isinstance(val[0], bytes), val[0]
self.assertEqual(val[0], b'\xc2\xa3')
def test_encode_latin1(self):
h = Headers({u'key': u'\xa3'}, encoding='latin1')
h = Headers({'key': '\xa3'}, encoding='latin1')
key, val = dict(h).popitem()
self.assertEqual(val[0], b'\xa3')
def test_encode_multiple(self):
h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
h = Headers({'key': ['\xa3']}, encoding='utf-8')
key, val = dict(h).popitem()
self.assertEqual(val[0], b'\xc2\xa3')

94
tests/test_http_request.py
View File

@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
self.assertFalse(p.headers is r.headers)
# headers must not be unicode
h = Headers({'key1': u'val1', u'key2': 'val2'})
h[u'newkey'] = u'newval'
h = Headers({'key1': 'val1', 'key2': 'val2'})
h['newkey'] = 'newval'
for k, v in h.items():
self.assertIsInstance(k, bytes)
for s in v:
@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
def test_url_encoding(self):
r = self.request_class(url=u"http://www.scrapy.org/price/£")
r = self.request_class(url="http://www.scrapy.org/price/£")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
def test_url_encoding_other(self):
# encoding affects only query part of URI, not path
# path part should always be UTF-8 encoded before percent-escaping
r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
def test_url_encoding_query(self):
r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
# should be same as above
r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
def test_url_encoding_query_latin1(self):
# encoding is used for encoding query-string before percent-escaping;
# path is still UTF-8 encoded before percent-escaping
r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
def test_url_encoding_nonutf8_untouched(self):
@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
# characters. Otherwise, in the future the IRI will be mapped to
# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
# URI from "http://www.example.org/r%E9sum%E9.html".
r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
def test_body(self):
@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
assert isinstance(r2.body, bytes)
self.assertEqual(r2.encoding, 'utf-8') # default encoding
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
assert isinstance(r3.body, bytes)
self.assertEqual(r3.body, b"Price: \xc2\xa3100")
r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
assert isinstance(r4.body, bytes)
self.assertEqual(r4.body, b"Price: \xa3100")
@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
# unicode url
r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
def test_copy(self):
@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
assert r4.dont_filter is False
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method=u"POST")
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
def test_immutable_attributes(self):
@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
def test_default_encoding_textual_data(self):
# using default encoding (utf-8)
data = {u'µ one': u'two', u'price': u'£ 100'}
data = {'µ one': 'two', 'price': '£ 100'}
r2 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'utf-8')
@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
def test_default_encoding_mixed_data(self):
# using default encoding (utf-8)
data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
r2 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'utf-8')
@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
def test_custom_encoding_textual_data(self):
data = {'price': u'£ 100'}
data = {'price': '£ 100'}
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
self.assertEqual(r3.encoding, 'latin1')
self.assertEqual(r3.body, b'price=%A3+100')
def test_multi_key_values(self):
# using multiples values for a single key
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.url, "http://www.example.com/this/post.php")
fs = _qs(req, to_unicode=True)
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
self.assertEqual(fs[u'test2'], [u'xxx µ'])
self.assertEqual(fs[u'six'], [u'seven'])
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
self.assertEqual(set(fs['one']), {'two', 'three'})
self.assertEqual(fs['test2'], ['xxx µ'])
self.assertEqual(fs['six'], ['seven'])
def test_from_response_post_nonascii_bytes_latin1(self):
response = _buildresponse(
@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.url, "http://www.example.com/this/post.php")
fs = _qs(req, to_unicode=True, encoding='latin1')
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
self.assertEqual(fs[u'test2'], [u'xxx µ'])
self.assertEqual(fs[u'six'], [u'seven'])
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
self.assertEqual(set(fs['one']), {'two', 'three'})
self.assertEqual(fs['test2'], ['xxx µ'])
self.assertEqual(fs['six'], ['seven'])
def test_from_response_post_nonascii_unicode(self):
response = _buildresponse(
u"""<form action="post.php" method="POST">
"""<form action="post.php" method="POST">
<input type="hidden" name="test £" value="val1">
<input type="hidden" name="test £" value="val2">
<input type="hidden" name="test2" value="xxx µ">
@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.url, "http://www.example.com/this/post.php")
fs = _qs(req, to_unicode=True)
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
self.assertEqual(fs[u'test2'], [u'xxx µ'])
self.assertEqual(fs[u'six'], [u'seven'])
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
self.assertEqual(set(fs['one']), {'two', 'three'})
self.assertEqual(fs['test2'], ['xxx µ'])
self.assertEqual(fs['six'], ['seven'])
def test_from_response_duplicate_form_key(self):
response = _buildresponse(
@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="clicked2">
</form>""")
req = self.request_class.from_response(
response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
response, clickdata={'name': 'clickable', 'value': 'clicked2'}
)
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
def test_from_response_unicode_clickdata(self):
response = _buildresponse(
u"""<form action="get.php" method="GET">
"""<form action="get.php" method="GET">
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
<input type="submit" name="price in \u20ac" value="\u20ac 2000">
<input type="hidden" name="poundsign" value="\u00a3">
<input type="hidden" name="eurosign" value="\u20ac">
</form>""")
req = self.request_class.from_response(
response, clickdata={u'name': u'price in \u00a3'}
response, clickdata={'name': 'price in \u00a3'}
)
fs = _qs(req, to_unicode=True)
self.assertTrue(fs[u'price in \u00a3'])
self.assertTrue(fs['price in \u00a3'])
def test_from_response_unicode_clickdata_latin1(self):
response = _buildresponse(
u"""<form action="get.php" method="GET">
"""<form action="get.php" method="GET">
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
<input type="submit" name="price in \u00a5" value="\u00a5 2000">
<input type="hidden" name="poundsign" value="\u00a3">
@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
</form>""",
encoding='latin1')
req = self.request_class.from_response(
response, clickdata={u'name': u'price in \u00a5'}
response, clickdata={'name': 'price in \u00a5'}
)
fs = _qs(req, to_unicode=True, encoding='latin1')
self.assertTrue(fs[u'price in \u00a5'])
self.assertTrue(fs['price in \u00a5'])
def test_from_response_multiple_forms_clickdata(self):
response = _buildresponse(
@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
</form>
""")
req = self.request_class.from_response(
response, formname='form2', clickdata={u'name': u'clickable'}
response, formname='form2', clickdata={'name': 'clickable'}
)
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
def test_from_response_unicode_xpath(self):
response = _buildresponse(b'<form name="\xd1\x8a"></form>')
r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
fs = _qs(r)
self.assertEqual(fs, {})
xpath = u"//form[@name='\u03b1']"
xpath = "//form[@name='\u03b1']"
self.assertRaisesRegex(ValueError, re.escape(xpath),
self.request_class.from_response,
response, formxpath=xpath)
@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
self._test_request(params=('value',))
self._test_request(params=('username', 'password'), methodname='login')
self._test_request(params=('response', ), methodresponse='login')
self._test_request(params=(u'pas£',), encoding='utf-8')
self._test_request(params=('pas£',), encoding='utf-8')
self._test_request(params=(None,), allow_none=1)
self.assertRaises(TypeError, self._test_request)
self.assertRaises(TypeError, self._test_request, params=(None,))
def test_latin1(self):
self._test_request(params=(u'pas£',), encoding='latin1')
self._test_request(params=('pas£',), encoding='latin1')
class JsonRequestTest(RequestTest):
@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):
def setUp(self):
warnings.simplefilter("always")
super(JsonRequestTest, self).setUp()
super().setUp()
def test_data(self):
r1 = self.request_class(url="http://www.example.com/")
@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):
def tearDown(self):
warnings.resetwarnings()
super(JsonRequestTest, self).tearDown()
super().tearDown()
if __name__ == "__main__":

74
tests/test_http_response.py
View File

@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):
response_class = TextResponse
def test_replace(self):
super(TextResponseTest, self).test_replace()
super().test_replace()
r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class(u"http://www.example.com/")
resp = self.response_class("http://www.example.com/")
self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
# make sure urls are converted to str
resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
assert isinstance(resp.url, str)
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
resp = self.response_class(u"http://www.example.com/price/\xa3",
resp = self.response_class("http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]})
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
resp = self.response_class(u"http://www.example.com/price/\xa3",
resp = self.response_class("http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
def test_unicode_body(self):
unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
'\u0442\u0435\u043a\u0441\u0442')
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
original_string = unicode_string.encode('cp1251')
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
def test_encoding(self):
r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]})
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
r3 = self.response_class("http://www.example.com", body=b"\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(r5._headers_encoding(), None)
self._assert_response_encoding(r5, "utf-8")
assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
self._assert_response_values(r1, 'utf-8', u"\xa3")
self._assert_response_values(r2, 'utf-8', u"\xa3")
self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
self._assert_response_values(r6, 'gb18030', u"\u2015")
self._assert_response_values(r7, 'gb18030', u"\u2015")
self._assert_response_values(r1, 'utf-8', "\xa3")
self._assert_response_values(r2, 'utf-8', "\xa3")
self._assert_response_values(r3, 'iso-8859-1', "\xa3")
self._assert_response_values(r6, 'gb18030', "\u2015")
self._assert_response_values(r7, 'gb18030', "\u2015")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
headers={"Content-type": ["text/html; charset=UKNOWN"]},
body=b"\xc2\xa3")
self.assertEqual(r._declared_encoding(), None)
self._assert_response_values(r, 'utf-8', u"\xa3")
self._assert_response_values(r, 'utf-8', "\xa3")
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
r = self.response_class("http://www.example.com",
body=b'\xff\xfeh\x00i\x00',
encoding='utf-16')
self._assert_response_values(r, 'utf-16', u"hi")
self._assert_response_values(r, 'utf-16', "hi")
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class("http://www.example.com",
@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
body=b"\xef\xbb\xbfWORD\xe3\xab")
self.assertEqual(r6.encoding, 'utf-8')
self.assertIn(r6.text, {
u'WORD\ufffd\ufffd', # w3lib < 1.19.0
u'WORD\ufffd', # w3lib >= 1.19.0
'WORD\ufffd\ufffd', # w3lib < 1.19.0
'WORD\ufffd', # w3lib >= 1.19.0
})
def test_bom_is_removed_from_body(self):
@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
self.assertEqual(response.encoding, 'utf-8')
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.text, 'WORD')
response = self.response_class(url, body=body)
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.text, 'WORD')
self.assertEqual(response.encoding, 'utf-8')
# Body caching sideeffect isn't triggered when encoding is declared in
@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
# body
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.encoding, 'utf-8')
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.text, 'WORD')
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.text, u'WORD')
self.assertEqual(response.text, 'WORD')
self.assertEqual(response.encoding, 'utf-8')
def test_replace_wrong_encoding(self):
"""Test invalid chars are replaced properly"""
r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
# XXX: Policy for replacing invalid chars may suffer minor variations
# but it should always contain the unicode replacement char (u'\ufffd')
assert u'\ufffd' in r.text, repr(r.text)
assert u'PREFIX' in r.text, repr(r.text)
assert u'SUFFIX' in r.text, repr(r.text)
# but it should always contain the unicode replacement char ('\ufffd')
assert '\ufffd' in r.text, repr(r.text)
assert 'PREFIX' in r.text, repr(r.text)
assert 'SUFFIX' in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class("http://example.com", encoding='utf-8',
body=b'\xf0<span>value</span>')
assert u'<span>value</span>' in r.text, repr(r.text)
assert '<span>value</span>' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
# assert u'\ufffd' in r.text, repr(r.text)
# assert '\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"
@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(
response.selector.xpath("//title/text()").getall(),
[u'Some page']
['Some page']
)
self.assertEqual(
response.selector.css("title::text").getall(),
[u'Some page']
['Some page']
)
self.assertEqual(
response.selector.re("Some (.*)</title>"),
[u'page']
['page']
)
def test_selector_shortcuts(self):
@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
resp1 = self.response_class(
'http://example.com',
encoding='utf8',
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
)
req = self._assert_followed_url(
resp1.css('a')[0],
@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
resp2 = self.response_class(
'http://example.com',
encoding='cp1251',
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
)
req = self._assert_followed_url(
resp2.css('a')[0],
@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
def test_body_as_unicode_deprecation_warning(self):
with catch_warnings(record=True) as warnings:
r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
self.assertEqual(r1.body_as_unicode(), u'Hello')
r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
self.assertEqual(r1.body_as_unicode(), 'Hello')
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
self.assertEqual(
response.selector.xpath("//elem/text()").getall(),
[u'value']
['value']
)
def test_selector_shortcuts(self):

44
tests/test_item.py
View File

@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
i['name'] = u'name'
self.assertEqual(i['name'], u'name')
i['name'] = 'name'
self.assertEqual(i['name'], 'name')
def test_init(self):
class TestItem(Item):
@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
i = TestItem()
self.assertRaises(KeyError, i.__getitem__, 'name')
i2 = TestItem(name=u'john doe')
self.assertEqual(i2['name'], u'john doe')
i2 = TestItem(name='john doe')
self.assertEqual(i2['name'], 'john doe')
i3 = TestItem({'name': u'john doe'})
self.assertEqual(i3['name'], u'john doe')
i3 = TestItem({'name': 'john doe'})
self.assertEqual(i3['name'], 'john doe')
i4 = TestItem(i3)
self.assertEqual(i4['name'], u'john doe')
self.assertEqual(i4['name'], 'john doe')
self.assertRaises(KeyError, TestItem, {'name': u'john doe',
'other': u'foo'})
self.assertRaises(KeyError, TestItem, {'name': 'john doe',
'other': 'foo'})
def test_invalid_field(self):
class TestItem(Item):
@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
number = Field()
i = TestItem()
i['name'] = u'John Doe'
i['name'] = 'John Doe'
i['number'] = 123
itemrepr = repr(i)
@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
i = TestItem()
self.assertRaises(KeyError, i.get_name)
i['name'] = u'lala'
self.assertEqual(i.get_name(), u'lala')
i.change_name(u'other')
i['name'] = 'lala'
self.assertEqual(i.get_name(), 'lala')
i.change_name('other')
self.assertEqual(i.get_name(), 'other')
def test_metaclass(self):
@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
values = Field()
i = TestItem()
i['name'] = u'John'
i['name'] = 'John'
self.assertEqual(list(i.keys()), ['name'])
self.assertEqual(list(i.values()), ['John'])
i['keys'] = u'Keys'
i['values'] = u'Values'
i['keys'] = 'Keys'
i['values'] = 'Values'
self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
def test_metaclass_with_fields_attribute(self):
class TestItem(Item):
fields = {'new': Field(default='X')}
item = TestItem(new=u'New')
item = TestItem(new='New')
self.assertSortedEqual(list(item.keys()), ['new'])
self.assertSortedEqual(list(item.values()), [u'New'])
self.assertSortedEqual(list(item.values()), ['New'])
def test_metaclass_inheritance(self):
class ParentItem(Item):
@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
i['name'] = u'John'
self.assertEqual(dict(i), {'name': u'John'})
i['name'] = 'John'
self.assertEqual(dict(i), {'name': 'John'})
def test_copy(self):
class TestItem(Item):
@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):
# requirement. When not done properly raises an error:
# TypeError: __class__ set to <class '__main__.MyItem'>
# defining 'MyItem' as <class '__main__.MyItem'>
super(MyItem, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
class DictItemTest(unittest.TestCase):

182
tests/test_linkextractors.py
View File

@ -31,31 +31,31 @@ class Base:
page4_url = 'http://example.com/page%204.html'
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url=page4_url, text=u'href with whitespaces'),
Link(url='http://www.google.com/something', text=''),
Link(url='http://example.com/innertag.html', text='inner tag'),
Link(url=page4_url, text='href with whitespaces'),
])
def test_extract_filter_allow(self):
lx = self.extractor_cls(allow=('sample', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
])
def test_extract_filter_allow_with_duplicates(self):
lx = self.extractor_cls(allow=('sample', ), unique=False)
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
])
@ -63,10 +63,10 @@ class Base:
lx = self.extractor_cls(allow=('sample', ), unique=False,
canonicalize=True)
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
])
@ -74,22 +74,22 @@ class Base:
lx = self.extractor_cls(allow=('sample',), unique=True,
canonicalize=True)
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
])
def test_extract_filter_allow_and_deny(self):
lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
])
def test_extract_filter_allowed_domains(self):
lx = self.extractor_cls(allow_domains=('google.com', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://www.google.com/something', text=u''),
Link(url='http://www.google.com/something', text=''),
])
def test_extraction_using_single_values(self):
@ -97,27 +97,27 @@ class Base:
lx = self.extractor_cls(allow='sample')
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo',
text='sample 3 repetition with fragment')
])
lx = self.extractor_cls(allow='sample', deny='3')
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(allow_domains='google.com')
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://www.google.com/something', text=u''),
Link(url='http://www.google.com/something', text=''),
])
lx = self.extractor_cls(deny_domains='example.com')
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://www.google.com/something', text=u''),
Link(url='http://www.google.com/something', text=''),
])
def test_nofollow(self):
@ -145,11 +145,11 @@ class Base:
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.org/about.html', text=u'About us'),
Link(url='http://example.org/follow.html', text=u'Follow this link'),
Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
Link(url='http://example.org/about.html', text='About us'),
Link(url='http://example.org/follow.html', text='Follow this link'),
Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
])
def test_matches(self):
@ -183,8 +183,8 @@ class Base:
def test_restrict_xpaths(self):
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
])
def test_restrict_xpaths_encoding(self):
@ -202,14 +202,14 @@ class Base:
lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
[Link(url='http://example.org/about.html', text='About us\xa3')])
def test_restrict_xpaths_with_html_entities(self):
html = b'<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
self.assertEqual(links,
[Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
[Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
def test_restrict_xpaths_concat_in_handle_data(self):
"""html entities cause SGMLParser to call handle_data hook twice"""
@ -217,22 +217,22 @@ class Base:
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
lx = self.extractor_cls(restrict_xpaths="//div")
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
[Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
fragment='', nofollow=False)])
def test_restrict_css(self):
lx = self.extractor_cls(restrict_css=('#subwrapper a',))
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample2.html', text=u'sample 2')
Link(url='http://example.com/sample2.html', text='sample 2')
])
def test_restrict_css_and_restrict_xpaths_together(self):
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
restrict_css=('#subwrapper + a', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
])
def test_area_tag_with_unicode_present(self):
@ -243,7 +243,7 @@ class Base:
lx.extract_links(response)
lx.extract_links(response)
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/foo', text=u'',
[Link(url='http://example.org/foo', text='',
fragment='', nofollow=False)])
def test_encoded_url(self):
@ -251,7 +251,7 @@ class Base:
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
])
def test_encoded_url_in_restricted_xpath(self):
@ -259,7 +259,7 @@ class Base:
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
lx = self.extractor_cls(restrict_xpaths="//div")
self.assertEqual(lx.extract_links(response), [
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
])
def test_ignored_extensions(self):
@ -268,7 +268,7 @@ class Base:
response = HtmlResponse("http://example.org/", body=html)
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.org/page.html', text=u'asd'),
Link(url='http://example.org/page.html', text='asd'),
])
# override denied extensions
@ -308,25 +308,25 @@ class Base:
page4_url = 'http://example.com/page%204.html'
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url=page4_url, text=u'href with whitespaces'),
Link(url='http://www.google.com/something', text=''),
Link(url='http://example.com/innertag.html', text='inner tag'),
Link(url=page4_url, text='href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample2.jpg', text=u''),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample2.jpg', text=''),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url=page4_url, text=u'href with whitespaces'),
Link(url='http://www.google.com/something', text=''),
Link(url='http://example.com/innertag.html', text='inner tag'),
Link(url=page4_url, text='href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)
@ -344,24 +344,24 @@ class Base:
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample1.html', text=''),
Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(tags="area")
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/sample1.html', text=u''),
Link(url='http://example.com/sample1.html', text=''),
])
lx = self.extractor_cls(tags="a")
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample2.jpg', text=u''),
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample2.jpg', text=''),
])
def test_tags_attrs(self):
@ -375,14 +375,14 @@ class Base:
lx = self.extractor_cls(tags='div', attrs='data-url')
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
])
lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
])
def test_xhtml(self):
@ -420,13 +420,13 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text='Dont follow this one',
fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
]
)
@ -436,13 +436,13 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
Link(url='http://example.com/nofollow.html', text='Dont follow this one',
fragment='', nofollow=True),
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
fragment='', nofollow=False),
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
]
)
@ -455,8 +455,8 @@ class Base:
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
])
def test_ftp_links(self):
@ -467,7 +467,7 @@ class Base:
response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
])
def test_pickle_extractor(self):
@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
])
def test_link_restrict_text(self):
@ -501,22 +501,22 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
# Simple text inclusion test
lx = self.extractor_cls(restrict_text='dog')
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
# Unique regex test
lx = self.extractor_cls(restrict_text=r'of.*dog')
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
# Multiple regex test
lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
def test_restrict_xpaths_with_html_entities(self):
super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
super().test_restrict_xpaths_with_html_entities()
def test_filteringlinkextractor_deprecation_warning(self):
"""Make sure the FilteringLinkExtractor deprecation warning is not

126
tests/test_loader.py
View File

@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_add_value_on_unknown_field(self):
il = TestItemLoader()
self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
def test_load_item_using_default_loader(self):
i = TestItem()
i['summary'] = u'lala'
i['summary'] = 'lala'
il = ItemLoader(item=i)
il.add_value('name', u'marta')
il.add_value('name', 'marta')
item = il.load_item()
assert item is i
self.assertEqual(item['summary'], [u'lala'])
self.assertEqual(item['name'], [u'marta'])
self.assertEqual(item['summary'], ['lala'])
self.assertEqual(item['name'], ['marta'])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
il.add_value('name', u'marta')
il.add_value('name', 'marta')
item = il.load_item()
self.assertEqual(item['name'], [u'Marta'])
self.assertEqual(item['name'], ['Marta'])
class InitializationTestMixin:
@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):
temp = Field()
def __init__(self, *args, **kwargs):
super(TempItem, self).__init__(self, *args, **kwargs)
super().__init__(self, *args, **kwargs)
self.setdefault('temp', 0.3)
class TempLoader(ItemLoader):
@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertRaises(RuntimeError, l.get_css, '#name::text')
def test_init_method_with_selector(self):
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_selector_css(self):
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_css('name', 'div::text')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_response(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_response_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('name', 'div::text')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.add_css('url', 'a::attr(href)')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
# combining/accumulating CSS selectors and XPath expressions
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
l.add_xpath('url', '//img/@src')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
def test_add_xpath_re(self):
l = TestItemLoader(response=self.response)
l.add_xpath('name', '//div/text()', re='ma')
self.assertEqual(l.get_output_value('name'), [u'Ma'])
self.assertEqual(l.get_output_value('name'), ['Ma'])
def test_replace_xpath(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath('name', '//p/text()')
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.replace_xpath('name', ['//p/text()', '//div/text()'])
self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
def test_get_xpath(self):
l = TestItemLoader(response=self.response)
self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
def test_replace_xpath_multi_fields(self):
l = TestItemLoader(response=self.response)
l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
def test_replace_xpath_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath('name', '//div/text()', re='ma')
self.assertEqual(l.get_output_value('name'), [u'Ma'])
self.assertEqual(l.get_output_value('name'), ['Ma'])
def test_add_css_re(self):
l = TestItemLoader(response=self.response)
l.add_css('name', 'div::text', re='ma')
self.assertEqual(l.get_output_value('name'), [u'Ma'])
self.assertEqual(l.get_output_value('name'), ['Ma'])
l.add_css('url', 'a::attr(href)', re='http://(.+)')
self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
def test_replace_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('name', 'div::text')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_css('name', 'p::text')
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.replace_css('name', ['p::text', 'div::text'])
self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
l.add_css('url', 'a::attr(href)', re='http://(.+)')
self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
l.replace_css('url', 'img::attr(src)')
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
def test_get_css(self):
l = TestItemLoader(response=self.response)
self.assertEqual(l.get_css('p::text'), [u'paragraph'])
self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
self.assertEqual(l.get_css('p::text'), ['paragraph'])
self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
[u'http://www.scrapy.org', u'/images/logo.png'])
['http://www.scrapy.org', '/images/logo.png'])
def test_replace_css_multi_fields(self):
l = TestItemLoader(response=self.response)
l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Marta'])
self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
def test_replace_css_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('url', 'a::attr(href)')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
class SubselectorLoaderTest(unittest.TestCase):
@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
self.assertEqual(l.get_output_value('name'), ['marta'])
self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
self.assertEqual(l.get_output_value('name_value'), ['marta'])
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
self.assertEqual(l.get_output_value('name'), [u'marta'])
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
self.assertEqual(l.get_output_value('name'), ['marta'])
self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
self.assertEqual(l.get_output_value('name_value'), ['marta'])
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
nl2 = nl1.nested_xpath('a')
l.add_xpath('url', '//footer/a/@href')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
nl1.replace_xpath('url', 'img/@src')
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
nl2.replace_xpath('url', '@href')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
def test_nested_ordering(self):
l = NestedItemLoader(response=self.response)
@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
l.add_xpath('url', '//footer/a/@href')
self.assertEqual(l.get_output_value('url'), [
u'/images/logo.png',
u'http://www.scrapy.org',
u'homepage',
u'http://www.scrapy.org',
'/images/logo.png',
'http://www.scrapy.org',
'homepage',
'http://www.scrapy.org',
])
def test_nested_load_item(self):
@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
assert item is nl1.item
assert item is nl2.item
self.assertEqual(item['name'], [u'marta'])
self.assertEqual(item['url'], [u'http://www.scrapy.org'])
self.assertEqual(item['image'], [u'/images/logo.png'])
self.assertEqual(item['name'], ['marta'])
self.assertEqual(item['url'], ['http://www.scrapy.org'])
self.assertEqual(item['image'], ['/images/logo.png'])
# Functions as processors

228
tests/test_loader_deprecated.py
View File

@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_load_item_using_default_loader(self):
i = TestItem()
i['summary'] = u'lala'
i['summary'] = 'lala'
il = ItemLoader(item=i)
il.add_value('name', u'marta')
il.add_value('name', 'marta')
item = il.load_item()
assert item is i
self.assertEqual(item['summary'], [u'lala'])
self.assertEqual(item['name'], [u'marta'])
self.assertEqual(item['summary'], ['lala'])
self.assertEqual(item['name'], ['marta'])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
il.add_value('name', u'marta')
il.add_value('name', 'marta')
item = il.load_item()
self.assertEqual(item['name'], [u'Marta'])
self.assertEqual(item['name'], ['Marta'])
def test_load_item_ignore_none_field_values(self):
def validate_sku(value):
@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
price_out = Compose(TakeFirst(), float)
sku_out = Compose(TakeFirst(), validate_sku)
valid_fragment = u'SKU: 1234'
invalid_fragment = u'SKU: not available'
valid_fragment = 'SKU: 1234'
invalid_fragment = 'SKU: not available'
sku_re = 'SKU: (.+)'
il = MyLoader(item={})
# Should not return "sku: None".
il.add_value('sku', [invalid_fragment], re=sku_re)
# Should not ignore empty values.
il.add_value('name', u'')
il.add_value('price', [u'0'])
il.add_value('name', '')
il.add_value('price', ['0'])
self.assertEqual(il.load_item(), {
'name': u'',
'name': '',
'price': 0.0,
})
il.replace_value('sku', [valid_fragment], re=sku_re)
self.assertEqual(il.load_item()['sku'], u'1234')
self.assertEqual(il.load_item()['sku'], '1234')
def test_self_referencing_loader(self):
class MyLoader(ItemLoader):
@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_add_value(self):
il = TestItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_collected_values('name'), [u'Marta'])
self.assertEqual(il.get_output_value('name'), [u'Marta'])
il.add_value('name', u'pepe')
self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
il.add_value('name', 'marta')
self.assertEqual(il.get_collected_values('name'), ['Marta'])
self.assertEqual(il.get_output_value('name'), ['Marta'])
il.add_value('name', 'pepe')
self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
# test add object value
il.add_value('summary', {'key': 1})
self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
il.add_value(None, u'Jim', lambda x: {'name': x})
self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
il.add_value(None, 'Jim', lambda x: {'name': x})
self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
def test_add_zero(self):
il = NameItemLoader()
@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_replace_value(self):
il = TestItemLoader()
il.replace_value('name', u'marta')
self.assertEqual(il.get_collected_values('name'), [u'Marta'])
self.assertEqual(il.get_output_value('name'), [u'Marta'])
il.replace_value('name', u'pepe')
self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
self.assertEqual(il.get_output_value('name'), [u'Pepe'])
il.replace_value('name', 'marta')
self.assertEqual(il.get_collected_values('name'), ['Marta'])
self.assertEqual(il.get_output_value('name'), ['Marta'])
il.replace_value('name', 'pepe')
self.assertEqual(il.get_collected_values('name'), ['Pepe'])
self.assertEqual(il.get_output_value('name'), ['Pepe'])
il.replace_value(None, u'Jim', lambda x: {'name': x})
self.assertEqual(il.get_collected_values('name'), [u'Jim'])
il.replace_value(None, 'Jim', lambda x: {'name': x})
self.assertEqual(il.get_collected_values('name'), ['Jim'])
def test_get_value(self):
il = NameItemLoader()
self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
self.assertEqual([u'foo'], il.get_collected_values('name'))
il.replace_value('name', u'name:bar', re=u'name:(.*)$')
self.assertEqual([u'bar'], il.get_collected_values('name'))
il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
self.assertEqual(['foo'], il.get_collected_values('name'))
il.replace_value('name', 'name:bar', re='name:(.*)$')
self.assertEqual(['bar'], il.get_collected_values('name'))
def test_iter_on_input_processor_input(self):
class NameFirstItemLoader(NameItemLoader):
name_in = TakeFirst()
il = NameFirstItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_collected_values('name'), [u'marta'])
il.add_value('name', 'marta')
self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
il.add_value('name', [u'marta', u'jose'])
self.assertEqual(il.get_collected_values('name'), [u'marta'])
il.add_value('name', ['marta', 'jose'])
self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
il.replace_value('name', u'marta')
self.assertEqual(il.get_collected_values('name'), [u'marta'])
il.replace_value('name', 'marta')
self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
il.replace_value('name', [u'marta', u'jose'])
self.assertEqual(il.get_collected_values('name'), [u'marta'])
il.replace_value('name', ['marta', 'jose'])
self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
il.add_value('name', u'marta')
il.add_value('name', [u'jose', u'pedro'])
self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
il.add_value('name', 'marta')
il.add_value('name', ['jose', 'pedro'])
self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
def test_map_compose_filter(self):
def filter_world(x):
@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'Mart'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['Mart'])
item = il.load_item()
self.assertEqual(item['name'], [u'Mart'])
self.assertEqual(item['name'], ['Mart'])
def test_default_input_processor(self):
il = DefaultedItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'mart'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['mart'])
def test_inherited_default_input_processor(self):
class InheritDefaultedItemLoader(DefaultedItemLoader):
pass
il = InheritDefaultedItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'mart'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['mart'])
def test_input_processor_inheritance(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(lambda v: v.lower())
il = ChildItemLoader()
il.add_value('url', u'HTTP://scrapy.ORG')
self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'Marta'])
il.add_value('url', 'HTTP://scrapy.ORG')
self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['Marta'])
class ChildChildItemLoader(ChildItemLoader):
url_in = MapCompose(lambda v: v.upper())
summary_in = MapCompose(lambda v: v)
il = ChildChildItemLoader()
il.add_value('url', u'http://scrapy.org')
self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'Marta'])
il.add_value('url', 'http://scrapy.org')
self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['Marta'])
def test_empty_map_compose(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose()
il = IdentityDefaultedItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'marta'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['marta'])
def test_identity_input_processor(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = Identity()
il = IdentityDefaultedItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'marta'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['marta'])
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'mARTA'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['mARTA'])
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
il = ChildDefaultedItemLoader()
il.add_value('name', u'marta')
self.assertEqual(il.get_output_value('name'), [u'MART'])
il.add_value('name', 'marta')
self.assertEqual(il.get_output_value('name'), ['MART'])
def test_output_processor_using_function(self):
il = TestItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class TakeFirstItemLoader(TestItemLoader):
name_out = u" ".join
name_out = " ".join
il = TakeFirstItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), u'Mar Ta')
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar Ta')
def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = MapCompose(float)
il = TestItemLoader()
il.add_value('name', [u'$10'])
il.add_value('name', ['$10'])
try:
float(u'$10')
float('$10')
except Exception as e:
expected_exc_str = str(e)
@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_output_processor_using_classes(self):
il = TestItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class TakeFirstItemLoader(TestItemLoader):
name_out = Join()
il = TakeFirstItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), u'Mar Ta')
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar Ta')
class TakeFirstItemLoader(TestItemLoader):
name_out = Join("<br>")
il = TakeFirstItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
def test_default_output_processor(self):
il = TestItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class LalaItemLoader(TestItemLoader):
default_output_processor = Identity()
il = LalaItemLoader()
il.add_value('name', [u'mar', u'ta'])
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
il.add_value('name', ['mar', 'ta'])
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
def test_loader_context_on_declaration(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor_with_args, key=u'val')
url_in = MapCompose(processor_with_args, key='val')
il = ChildItemLoader()
il.add_value('url', u'text')
il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
il.replace_value('url', u'text2')
il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_loader_context_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader(key=u'val')
il.add_value('url', u'text')
il = ChildItemLoader(key='val')
il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
il.replace_value('url', u'text2')
il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_loader_context_on_assign(self):
@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader()
il.context['key'] = u'val'
il.add_value('url', u'text')
il.context['key'] = 'val'
il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
il.replace_value('url', u'text2')
il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_item_passed_to_input_processor_functions(self):
@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
it = TestItem(name='marta')
il = ChildItemLoader(item=it)
il.add_value('url', u'text')
il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['marta'])
il.replace_value('url', u'text2')
il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['marta'])
def test_compose_processor(self):
@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
il.add_value('name', [u'marta', u'other'])
self.assertEqual(il.get_output_value('name'), u'Mart')
il.add_value('name', ['marta', 'other'])
self.assertEqual(il.get_output_value('name'), 'Mart')
item = il.load_item()
self.assertEqual(item['name'], u'Mart')
self.assertEqual(item['name'], 'Mart')
def test_partial_processor(self):
def join(values, sep=None, loader_context=None, ignored=None):
@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
summary_out = Compose(partial(join, ignored='foo'))
il = TestItemLoader()
il.add_value('name', [u'rabbit', u'hole'])
il.add_value('url', [u'rabbit', u'hole'])
il.add_value('summary', [u'rabbit', u'hole'])
il.add_value('name', ['rabbit', 'hole'])
il.add_value('url', ['rabbit', 'hole'])
il.add_value('summary', ['rabbit', 'hole'])
item = il.load_item()
self.assertEqual(item['name'], u'rabbit+hole')
self.assertEqual(item['url'], u'rabbit.hole')
self.assertEqual(item['summary'], u'rabbithole')
self.assertEqual(item['name'], 'rabbit+hole')
self.assertEqual(item['url'], 'rabbit.hole')
self.assertEqual(item['summary'], 'rabbithole')
def test_error_input_processor(self):
class TestItem(Item):
@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
[u'marta', u'other'])
['marta', 'other'])
def test_error_output_processor(self):
class TestItem(Item):
@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = Compose(Join(), float)
il = TestItemLoader()
il.add_value('name', u'marta')
il.add_value('name', 'marta')
with self.assertRaises(ValueError):
il.load_item()
@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
[u'marta', u'other'], Compose(float))
['marta', 'other'], Compose(float))
class InitializationFromDictTest(unittest.TestCase):
@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):
class TempDict(dict):
def __init__(self, *args, **kwargs):
super(TempDict, self).__init__(self, *args, **kwargs)
super().__init__(self, *args, **kwargs)
self.setdefault('temp', 0.3)
class TempLoader(ItemLoader):
@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
def test_join(self):
proc = Join()
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
self.assertEqual(proc(['hello', 'world']), u'hello world')
self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
self.assertEqual(proc(['hello', 'world']), 'hello world')
self.assertIsInstance(proc(['hello', 'world']), str)
def test_compose(self):
@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
def filter_world(x):
return None if x == 'world' else x
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
['HELLO', 'THIS', 'IS', 'SCRAPY'])
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
proc = MapCompose(filter_world, str.upper)

14
tests/test_logformatter.py
View File

@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
def test_dropped(self):
item = {}
exception = Exception(u"\u2018")
exception = Exception("\u2018")
response = Response("http://www.example.com")
logkws = self.formatter.dropped(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
self.assertEqual(lines, ["Dropped: \u2018", '{}'])
def test_item_error(self):
# In practice, the complete traceback is shown by passing the
@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
response = Response("http://www.example.com")
logkws = self.formatter.item_error(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline, u"Error processing {'key': 'value'}")
self.assertEqual(logline, "Error processing {'key': 'value'}")
def test_spider_error(self):
# In practice, the complete traceback is shown by passing the
@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
def test_scraped(self):
item = CustomItem()
item['name'] = u'\xa3'
item['name'] = '\xa3'
response = Response("http://www.example.com")
logkws = self.formatter.scraped(item, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
class LogFormatterSubclass(LogFormatter):
def crawled(self, request, response, spider):
kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
kwargs = super().crawled(request, response, spider)
CRAWLEDMSG = (
u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
)
log_args = kwargs['args']
log_args['flags'] = str(request.flags)

8
tests/test_mail.py
View File

@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
self.catched_msg = dict(**kwargs)
def test_send_utf8(self):
subject = u'sübjèçt'
body = u'bödÿ-àéïöñß'
subject = 'sübjèçt'
body = 'bödÿ-àéïöñß'
mailsender = MailSender(debug=True)
mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
charset='utf-8', _callback=self._catch_mail_sent)
@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
def test_send_attach_utf8(self):
subject = u'sübjèçt'
body = u'bödÿ-àéïöñß'
subject = 'sübjèçt'
body = 'bödÿ-àéïöñß'
attach = BytesIO()
attach.write(body.encode('utf-8'))
attach.seek(0)

2
tests/test_middleware.py
View File

@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):
return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
def _add_middleware(self, mw):
super(TestMiddlewareManager, self)._add_middleware(mw)
super()._add_middleware(mw)
if hasattr(mw, 'process'):
self.methods['process'].append(mw.process)

12
tests/test_pipeline_media.py
View File

@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
class MockedMediaPipeline(MediaPipeline):
def __init__(self, *args, **kwargs):
super(MockedMediaPipeline, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._mockcalled = []
def download(self, request, info):
self._mockcalled.append('download')
return super(MockedMediaPipeline, self).download(request, info)
return super().download(request, info)
def media_to_download(self, request, info):
self._mockcalled.append('media_to_download')
if 'result' in request.meta:
return request.meta.get('result')
return super(MockedMediaPipeline, self).media_to_download(request, info)
return super().media_to_download(request, info)
def get_media_requests(self, item, info):
self._mockcalled.append('get_media_requests')
@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):
def media_downloaded(self, response, request, info):
self._mockcalled.append('media_downloaded')
return super(MockedMediaPipeline, self).media_downloaded(response, request, info)
return super().media_downloaded(response, request, info)
def media_failed(self, failure, request, info):
self._mockcalled.append('media_failed')
return super(MockedMediaPipeline, self).media_failed(failure, request, info)
return super().media_failed(failure, request, info)
def item_completed(self, results, item, info):
self._mockcalled.append('item_completed')
item = super(MockedMediaPipeline, self).item_completed(results, item, info)
item = super().item_completed(results, item, info)
item['results'] = results
return item

2
tests/test_request_left.py
View File

@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):
name = 'signal_catcher'
def __init__(self, crawler, url, *args, **kwargs):
super(SignalCatcherSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
crawler.signals.connect(self.on_request_left,
signal=request_left_downloader)
self.caught_times = 0

10
tests/test_responsetypes.py
View File

@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
mappings = [
(b'attachment; filename="data.xml"', XmlResponse),
(b'attachment; filename=data.xml', XmlResponse),
(u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
(u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
(u'attachment;filename=data高.doc'.encode('gbk'), Response),
(u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
(u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
('attachment;filename=data高.doc'.encode('gbk'), Response),
('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
]
for source, cls in mappings:

16
tests/test_robotstxt_interface.py
View File

@ -93,7 +93,7 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
def test_unicode_url_and_useragent(self):
robotstxt_robotstxt_body = u"""
robotstxt_robotstxt_body = """
User-Agent: *
Disallow: /admin/
Disallow: /static/
@ -107,17 +107,17 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://site.local/", "*"))
self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
self.assertFalse(rp.allowed("https://site.local/static/", "*"))
self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import PythonRobotParser
super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
super()._setUp(PythonRobotParser)
def test_length_based_precedence(self):
raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import ReppyRobotParser
super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
super()._setUp(ReppyRobotParser)
def test_order_based_precedence(self):
raise unittest.SkipTest("Reppy does not support order based directives precedence.")
@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import RerpRobotParser
super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
raise unittest.SkipTest("Rerp does not support length based directives precedence.")
@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import ProtegoRobotParser
super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
raise unittest.SkipTest("Protego does not support order based directives precedence.")

4
tests/test_scheduler.py
View File

@ -53,7 +53,7 @@ class MockCrawler(Crawler):
JOBDIR=jobdir,
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
)
super(MockCrawler, self).__init__(Spider, settings)
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())
@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
def __init__(self, start_urls):
self.start_urls = start_urls
super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
super().__init__(name='StartUrlsSpider')
def parse(self, response):
pass

32
tests/test_selector.py
View File

@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
)
self.assertEqual(
[x.get() for x in sel.xpath("//input[@name='a']/@name")],
[u'a']
['a']
)
self.assertEqual(
[x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
[u'12.0']
['12.0']
)
self.assertEqual(
sel.xpath("concat('xpath', 'rules')").getall(),
[u'xpathrules']
['xpathrules']
)
self.assertEqual(
[x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
[u'12']
['12']
)
def test_root_base_url(self):
@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'xml')
self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
['<div><img src="a.jpg"><p>Hello</p></img></div>'])
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'html')
self.assertEqual(sel.xpath("//div").getall(),
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
['<div><img src="a.jpg"><p>Hello</p></div>'])
def test_http_header_encoding_precedence(self):
# u'\xa3' = pound symbol in unicode
# u'\xc2\xa3' = pound symbol in utf-8
# u'\xa3' = pound symbol in latin-1 (iso-8859-1)
# '\xa3' = pound symbol in unicode
# '\xc2\xa3' = pound symbol in utf-8
# '\xa3' = pound symbol in latin-1 (iso-8859-1)
meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
head = u'<head>' + meta + u'</head>'
body_content = u'<span id="blank">\xa3</span>'
body = u'<body>' + body_content + u'</body>'
html = u'<html>' + head + body + u'</html>'
meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
head = '<head>' + meta + '</head>'
body_content = '<span id="blank">\xa3</span>'
body = '<body>' + body_content + '</body>'
html = '<html>' + head + body + '</html>'
encoding = 'utf-8'
html_utf8 = html.encode(encoding)
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
x = Selector(response)
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
def test_selector_bad_args(self):
with self.assertRaisesRegex(ValueError, 'received both response and text'):
Selector(TextResponse(url='http://example.com', body=b''), text=u'')
Selector(TextResponse(url='http://example.com', body=b''), text='')

12
tests/test_spider.py
View File

@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
output = list(spider._parse(response))
self.assertEqual(len(output), 2, iterator)
self.assertEqual(output, [
{'loc': [u'http://www.example.com/Special-Offers.html'],
'updated': [u'2009-08-16'],
'custom': [u'fuu'],
'other': [u'bar']},
{'loc': ['http://www.example.com/Special-Offers.html'],
'updated': ['2009-08-16'],
'custom': ['fuu'],
'other': ['bar']},
{'loc': [],
'updated': [u'2009-08-16'],
'other': [u'foo'],
'updated': ['2009-08-16'],
'other': ['foo'],
'custom': []},
], iterator)

2
tests/test_spidermiddleware_httperror.py
View File

@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):
bypass_status_codes = set()
def __init__(self, *args, **kwargs):
super(_HttpErrorSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.start_urls = [
self.mockserver.url("/status?n=200"),
self.mockserver.url("/status?n=404"),

4
tests/test_utils_conf.py
View File

@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_URI_PARAMS": (1, 2, 3, 4),
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"indent": 42,
"store_empty": True,
"uri_params": (1, 2, 3, 4),
"batch_item_count": 2,
})
def test_feed_complete_default_values_from_settings_non_empty(self):
@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_EXPORT_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"indent": 42,
"store_empty": True,
"uri_params": None,
"batch_item_count": 2,
})

90
tests/test_utils_iterators.py
View File

@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
body = u"""<?xml version="1.0" encoding="UTF-8"?>
body = """<?xml version="1.0" encoding="UTF-8"?>
<þingflokkar>
<þingflokkur id="26">
<heiti />
@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
):
attrs = []
for x in self.xmliter(r, u'þingflokkur'):
for x in self.xmliter(r, 'þingflokkur'):
attrs.append((x.attrib['id'],
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
x.xpath('./tímabil/fyrstaþing/text()').getall()))
self.assertEqual(attrs,
[(u'26', [u'-'], [u'80']),
(u'21', [u'Ab'], [u'76']),
(u'27', [u'A'], [u'27'])])
[('26', ['-'], ['80']),
('21', ['Ab'], ['76']),
('27', ['A'], ['27'])])
def test_xmliter_text(self):
body = (
@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
)
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
[['one'], ['two']])
def test_xmliter_namespaces(self):
body = b"""
@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse('http://www.example.com', body=body)
self.assertEqual(
next(self.xmliter(response, 'item')).get(),
u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
)
@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
result = [row for row in csv]
self.assertEqual(result,
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_quotechar(self):
body1 = get_testdata('feeds', 'feed-sample6.csv')
@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
csv1 = csviter(response1, quotechar="'")
self.assertEqual([row for row in csv1],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
response2 = TextResponse(url="http://example.com/", body=body2)
csv2 = csviter(response2, delimiter="|", quotechar="'")
self.assertEqual([row for row in csv2],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_wrong_quotechar(self):
body = get_testdata('feeds', 'feed-sample6.csv')
@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
{u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
{u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
{u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
[{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
{"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
{"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
{"'id'": "4", "'name'": "'empty'", "'value'": ""}])
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_headers(self):
sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
{'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_falserow(self):
body = get_testdata('feeds', 'feed-sample3.csv')
@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
{'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_exception(self):
body = get_testdata('feeds', 'feed-sample3.csv')
@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'latin1', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
{'id': '1', 'name': 'latin1', 'value': 'test'},
{'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
]
)
@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'cp852', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
{'id': '1', 'name': 'cp852', 'value': 'test'},
{'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
]
)

16
tests/test_utils_python.py
View File

@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
class ToUnicodeTest(unittest.TestCase):
def test_converting_an_utf8_encoded_string_to_unicode(self):
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
def test_converting_a_latin_1_encoded_string_to_unicode(self):
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_unicode, 423)
@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
def test_errors_argument(self):
self.assertEqual(
to_unicode(b'a\xedb', 'utf-8', errors='replace'),
u'a\ufffdb'
'a\ufffdb'
)
class ToBytesTest(unittest.TestCase):
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
def test_errors_argument(self):
self.assertEqual(
to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
to_bytes('a\ufffdb', 'latin-1', errors='replace'),
b'a?b'
)
@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
assert binary_is_text(b"hello")
def test_utf_16_strings_contain_null_bytes(self):
assert binary_is_text(u"hello".encode('utf-16'))
assert binary_is_text("hello".encode('utf-16'))
def test_one_with_encoding(self):
assert binary_is_text(b"<div>Price \xa3</div>")

2
tests/test_utils_reqser.py
View File

@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
method="POST",
body=b"some body",
headers={'content-encoding': 'text/html; charset=latin-1'},
cookies={'currency': u'руб'},
cookies={'currency': 'руб'},
encoding='latin-1',
priority=20,
meta={'a': 'b'},

4
tests/test_utils_template.py
View File

@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
def test_simple_render(self):
context = dict(project_name='proj', name='spi', classname='TheSpider')
template = u'from ${project_name}.spiders.${name} import ${classname}'
rendered = u'from proj.spiders.spi import TheSpider'
template = 'from ${project_name}.spiders.${name} import ${classname}'
rendered = 'from proj.spiders.spi import TheSpider'
template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
render_path = os.path.join(self.tmp_path, 'templ.py')

1
tox.ini
View File

@ -12,6 +12,7 @@ deps =
-ctests/constraints.txt
-rtests/requirements-py3.txt
# Extras
boto3>=1.13.0
botocore>=1.3.23
Pillow>=3.4.2
passenv =