RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into http2

This commit is contained in:
Adrián Chaves 2020-10-06 21:05:10 +02:00 committed by GitHub
parent 6e8d20a07a b1255b016a
commit 269fe35d6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 675 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -16,6 +16,8 @@ htmlcov/
.coverage.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
/tests/keys/localhost.key
# Windows
Thumbs.db

6
conftest.py
View File

@ -2,6 +2,8 @@ from pathlib import Path
import pytest
from tests.keys import generate_keys
def _py_files(folder):
return (str(p) for p in Path(folder).rglob('*.py'))
@ -51,3 +53,7 @@ def reactor_pytest(request):
def only_asyncio(request, reactor_pytest):
if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
pytest.skip('This test is only run with --reactor=asyncio')
# Generate localhost certificate files, needed by some tests
generate_keys()

2
docs/contributing.rst
View File

@ -140,7 +140,7 @@ original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
not considered rude as long as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch

1
docs/index.rst
View File

@ -78,7 +78,6 @@ Basic concepts
topics/settings
topics/exceptions
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.

4
docs/intro/install.rst
View File

@ -211,8 +211,8 @@ PyPy
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
This means that these dependencies will be built during installation.
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,

5
docs/topics/feed-exports.rst
View File

@ -303,6 +303,9 @@ For instance::
'store_empty': False,
'fields': None,
'indent': 4,
'item_export_kwargs': {
'export_empty_fields': True,
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).

2
docs/topics/jobs.rst
View File

@ -65,7 +65,7 @@ Cookies expiration
------------------
Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
scheduled may no longer work. This won't be an issue if your spider doesn't rely
on cookies.

21
docs/topics/link-extractors.rst
View File

@ -10,12 +10,19 @@ The ``__init__`` method of
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
list of matching :class:`scrapy.link.Link` objects from a
list of matching :class:`~scrapy.link.Link` objects from a
:class:`~scrapy.http.Response` object.
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
extractors in regular spiders.
through a set of :class:`~scrapy.spiders.Rule` objects.
You can also use link extractors in regular spiders. For example, you can instantiate
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
variable in your spider, and use it from your spider callbacks::
def parse(self, response):
for link in self.link_extractor.extract_links(response):
yield Request(link.url, callback=self.parse)
.. _topics-link-extractors-ref:
@ -145,4 +152,12 @@ LxmlLinkExtractor
.. automethod:: extract_links
Link
----
.. module:: scrapy.link
:synopsis: Link from link extractors
.. autoclass:: Link
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

2
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:

2
scrapy/commands/runspider.py
View File

@ -11,7 +11,7 @@ def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext != '.py':
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path

55
scrapy/core/downloader/handlers/s3.py
View File

@ -2,41 +2,20 @@ from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import create_instance
def _get_boto_connection():
from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, method, bucket, key, headers, *args, **kwargs):
return headers
class _v20_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
def _mexe(self, http_request, *args, **kwargs):
http_request.authorize(connection=self)
return http_request.headers
try:
import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
_S3Connection = _v20_S3Connection
return _S3Connection
class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
@ -51,23 +30,15 @@ class S3DownloadHandler:
self.anon = kw.get('anon')
self._signer = None
if is_botocore():
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
import botocore.auth
import botocore.credentials
kw.pop('anon', None)
if kw:
raise TypeError(f'Unexpected keyword arguments: {kw}')
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
_http_handler = create_instance(
objcls=httpdownloadhandler,

27
scrapy/core/downloader/tls.py
View File

@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
@ -28,13 +27,6 @@ openssl_methods = {
}
if twisted_version < (17, 0, 0):
from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
else:
def set_tlsext_host_name(connection, hostNameBytes):
connection.set_tlsext_host_name(hostNameBytes)
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
@ -52,21 +44,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL.SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
connection.set_tlsext_host_name(self._hostnameBytes)
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
else:
logger.debug('SSL connection to %s using cipher %s',
self._hostnameASCII,
connection.get_cipher_name(),
)
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),

4
scrapy/crawler.py
View File

@ -180,9 +180,9 @@ class CrawlerRunner:
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
:class:`~scrapy.spiders.Spider` subclass or string
:param list args: arguments to initialize the spider
:param args: arguments to initialize the spider
:param dict kwargs: keyword arguments to initialize the spider
:param kwargs: keyword arguments to initialize the spider
"""
if isinstance(crawler_or_spidercls, Spider):
raise ValueError(

2
scrapy/downloadermiddlewares/cookies.py
View File

@ -74,7 +74,7 @@ class CookiesMiddleware:
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
if not cookie.get(key):
if cookie.get(key) is None:
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))

7
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -13,7 +13,12 @@ class HttpProxyMiddleware:
self.auth_encoding = auth_encoding
self.proxies = {}
for type_, url in getproxies().items():
self.proxies[type_] = self._get_proxy(url, type_)
try:
self.proxies[type_] = self._get_proxy(url, type_)
# some values such as '/var/run/docker.sock' can't be parsed
# by _parse_proxy and as such should be skipped
except ValueError:
continue
@classmethod
def from_crawler(cls, crawler):

36
scrapy/extensions/feedexport.py
View File

@ -19,7 +19,7 @@ from zope.interface import implementer, Interface
from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
@ -120,22 +120,19 @@ class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
feed_options=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
else:
import boto
self.connect_s3 = boto.connect_s3
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
@ -154,18 +151,10 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file):
file.seek(0)
if self.is_botocore:
kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file,
**kwargs)
else:
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
kwargs = {'policy': self.acl} if self.acl else {}
key.set_contents_from_file(file, **kwargs)
key.close()
kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file,
**kwargs)
file.close()
@ -360,6 +349,7 @@ class FeedExporter:
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
**feed_options['item_export_kwargs'],
)
slot = _FeedSlot(
file=file,

17
scrapy/link.py
View File

@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst
class Link:
"""Link objects represent an extracted link by the LinkExtractor."""
"""Link objects represent an extracted link by the LinkExtractor.
Using the anchor tag sample below to illustrate the parameters::
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
:param url: the absolute url being linked to in the anchor tag.
From the sample, this is ``https://example.com/nofollow.html``.
:param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
:param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
:param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
of the anchor tag.
"""
__slots__ = ['url', 'text', 'fragment', 'nofollow']

97
scrapy/pipelines/files.py
View File

@ -11,7 +11,6 @@ import os
import time
from collections import defaultdict
from contextlib import suppress
from email.utils import mktime_tz, parsedate_tz
from ftplib import FTP
from io import BytesIO
from urllib.parse import urlparse
@ -23,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
@ -91,86 +90,54 @@ class S3FilesStore:
}
def __init__(self, uri):
self.is_botocore = is_botocore()
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY
)
else:
from boto.s3.connection import S3Connection
self.S3Connection = S3Connection
if not is_botocore_available():
raise NotConfigured('missing botocore library')
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY
)
if not uri.startswith("s3://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split('/', 1)
def stat_file(self, path, info):
def _onsuccess(boto_key):
if self.is_botocore:
checksum = boto_key['ETag'].strip('"')
last_modified = boto_key['LastModified']
modified_stamp = time.mktime(last_modified.timetuple())
else:
checksum = boto_key.etag.strip('"')
last_modified = boto_key.last_modified
modified_tuple = parsedate_tz(last_modified)
modified_stamp = int(mktime_tz(modified_tuple))
checksum = boto_key['ETag'].strip('"')
last_modified = boto_key['LastModified']
modified_stamp = time.mktime(last_modified.timetuple())
return {'checksum': checksum, 'last_modified': modified_stamp}
return self._get_boto_key(path).addCallback(_onsuccess)
def _get_boto_bucket(self):
# disable ssl (is_secure=False) because of this python bug:
# https://bugs.python.org/issue5103
c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
return c.get_bucket(self.bucket, validate=False)
def _get_boto_key(self, path):
key_name = f'{self.prefix}{path}'
if self.is_botocore:
return threads.deferToThread(
self.s3_client.head_object,
Bucket=self.bucket,
Key=key_name)
else:
b = self._get_boto_bucket()
return threads.deferToThread(b.get_key, key_name)
return threads.deferToThread(
self.s3_client.head_object,
Bucket=self.bucket,
Key=key_name)
def persist_file(self, path, buf, info, meta=None, headers=None):
"""Upload file to S3 storage"""
key_name = f'{self.prefix}{path}'
buf.seek(0)
if self.is_botocore:
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
h.update(headers)
return threads.deferToThread(
k.set_contents_from_string, buf.getvalue(),
headers=h, policy=self.POLICY)
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
def _headers_to_botocore_kwargs(self, headers):
""" Convert headers to botocore keyword agruments.

34
scrapy/spiders/crawl.py
View File

@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst
"""
import copy
import warnings
from typing import Sequence
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils.python import get_func_args
from scrapy.utils.spider import iterate_spider_output
@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor()
class Rule:
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
process_links=None, process_request=None, errback=None):
def __init__(
self,
link_extractor=None,
callback=None,
cb_kwargs=None,
follow=None,
process_links=None,
process_request=None,
errback=None,
):
self.link_extractor = link_extractor or _default_link_extractor
self.callback = callback
self.errback = errback
self.cb_kwargs = cb_kwargs or {}
self.process_links = process_links or _identity
self.process_request = process_request or _identity_process_request
self.process_request_argcount = None
self.follow = follow if follow is not None else not callback
def _compile(self, spider):
@ -53,22 +57,6 @@ class Rule:
self.errback = _get_method(self.errback, spider)
self.process_links = _get_method(self.process_links, spider)
self.process_request = _get_method(self.process_request, spider)
self.process_request_argcount = len(get_func_args(self.process_request))
if self.process_request_argcount == 1:
warnings.warn(
"Rule.process_request should accept two arguments "
"(request, response), accepting only one is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def _process_request(self, request, response):
"""
Wrapper around the request processing function to maintain backward
compatibility with functions that do not take a Response object
"""
args = [request] if self.process_request_argcount == 1 else [request, response]
return self.process_request(*args)
class CrawlSpider(Spider):
@ -111,7 +99,7 @@ class CrawlSpider(Spider):
for link in rule.process_links(links):
seen.add(link)
request = self._build_request(rule_index, link)
yield rule._process_request(request, response)
yield rule.process_request(request, response)
def _callback(self, response):
rule = self._rules[response.meta['rule']]

23
scrapy/utils/boto.py
View File

@ -1,11 +1,32 @@
"""Boto/botocore helpers"""
import warnings
from scrapy.exceptions import NotConfigured
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
def is_botocore():
""" Returns True if botocore is available, otherwise raises NotConfigured. Never returns False.
Previously, when boto was supported in addition to botocore, this returned False if boto was available
but botocore wasn't.
"""
message = (
'is_botocore() is deprecated and always returns True or raises an Exception, '
'so it cannot be used for checking if boto is available instead of botocore. '
'You can use scrapy.utils.boto.is_botocore_available() to check if botocore '
'is available.'
)
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
try:
import botocore # noqa: F401
return True
except ImportError:
raise NotConfigured('missing botocore library')
def is_botocore_available():
try:
import botocore # noqa: F401
return True
except ImportError:
return False

1
scrapy/utils/conf.py
View File

@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
out.setdefault("item_export_kwargs", dict())
if settings["FEED_EXPORT_INDENT"] is None:
out.setdefault("indent", None)
else:

32
scrapy/utils/iterators.py
View File

@ -22,25 +22,41 @@ def xmliter(obj, nodename):
"""
nodename_patt = re.escape(nodename)
HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S)
DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S)
END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
text = _body_or_str(obj)
header_start = re.search(HEADER_START_RE, text)
header_start = header_start.group(1).strip() if header_start else ''
header_end = re_rsearch(HEADER_END_RE, text)
header_end = text[header_end[1]:].strip() if header_end else ''
document_header = re.search(DOCUMENT_HEADER_RE, text)
document_header = document_header.group().strip() if document_header else ''
header_end_idx = re_rsearch(HEADER_END_RE, text)
header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
namespaces = {}
if header_end:
for tagname in reversed(re.findall(END_TAG_RE, header_end)):
tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S)
if tag:
namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
r = re.compile(fr'<{nodename_patt}[\s>].*?</{nodename_patt}>', re.DOTALL)
for match in r.finditer(text):
nodetext = header_start + match.group() + header_end
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
nodetext = (
document_header
+ match.group().replace(
nodename,
f'{nodename} {" ".join(namespaces.values())}',
1
)
+ header_end
)
yield Selector(text=nodetext, type='xml')
def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
from lxml import etree
reader = _StreamReader(obj)
tag = f'{{{namespace}}}{nodename}'if namespace else nodename
tag = f'{{{namespace}}}{nodename}' if namespace else nodename
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
for _, node in iterable:

38
scrapy/utils/test.py
View File

@ -10,17 +10,7 @@ from unittest import mock
from importlib import import_module
from twisted.trial.unittest import SkipTest
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
def assert_aws_environ():
"""Asserts the current environment is suitable for running AWS testsi.
Raises SkipTest with the reason if it's not.
"""
skip_if_no_boto()
if 'AWS_ACCESS_KEY_ID' not in os.environ:
raise SkipTest("AWS keys not found")
from scrapy.utils.boto import is_botocore_available
def assert_gcs_environ():
@ -29,30 +19,8 @@ def assert_gcs_environ():
def skip_if_no_boto():
try:
is_botocore()
except NotConfigured as e:
raise SkipTest(e)
def get_s3_content_and_delete(bucket, path, with_key=False):
""" Get content from s3 key, and delete key afterwards.
"""
if is_botocore():
import botocore.session
session = botocore.session.get_session()
client = session.create_client('s3')
key = client.get_object(Bucket=bucket, Key=path)
content = key['Body'].read()
client.delete_object(Bucket=bucket, Key=path)
else:
import boto
# assuming boto=2.2.2
bucket = boto.connect_s3().get_bucket(bucket, validate=False)
key = bucket.get_key(path)
content = key.get_contents_as_string()
bucket.delete_key(path)
return (content, key) if with_key else content
if not is_botocore_available():
raise SkipTest('missing botocore library')
def get_gcs_content_and_delete(bucket, path):

63
tests/keys/__init__.py Normal file
View File

@ -0,0 +1,63 @@
import os
from datetime import datetime, timedelta
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.hashes import SHA256
from cryptography.hazmat.primitives.serialization import (
Encoding,
NoEncryption,
PrivateFormat,
)
from cryptography.x509 import (
CertificateBuilder,
DNSName,
Name,
NameAttribute,
random_serial_number,
SubjectAlternativeName,
)
from cryptography.x509.oid import NameOID
# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate
def generate_keys():
folder = os.path.dirname(__file__)
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend(),
)
with open(os.path.join(folder, 'localhost.key'), "wb") as f:
f.write(
key.private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
)
)
subject = issuer = Name(
[
NameAttribute(NameOID.COUNTRY_NAME, u"IE"),
NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"),
NameAttribute(NameOID.COMMON_NAME, u"localhost"),
]
)
cert = (
CertificateBuilder()
.subject_name(subject)
.issuer_name(issuer)
.public_key(key.public_key())
.serial_number(random_serial_number())
.not_valid_before(datetime.utcnow())
.not_valid_after(datetime.utcnow() + timedelta(days=10))
.add_extension(
SubjectAlternativeName([DNSName(u"localhost")]),
critical=False,
)
.sign(key, SHA256(), default_backend())
)
with open(os.path.join(folder, 'localhost.crt'), "wb") as f:
f.write(cert.public_bytes(Encoding.PEM))

20
tests/keys/localhost.crt
View File

@ -1,20 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL
BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j
YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE
BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK
2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe
pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH
wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7
56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN
cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB
o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU
VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ
VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX
FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf
2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE
W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb
dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA==
-----END CERTIFICATE-----

28
tests/keys/localhost.key
View File

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w
mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM
39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT
yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2
XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM
DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc
niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT
S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi
X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1
QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN
HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W
birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p
hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis
vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o
8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq
12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi
M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo
gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8
kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM
kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy
kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl
xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l
VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX
abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV
SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv
PWuPk5VX+qF78JhUhPO3nfnu
-----END PRIVATE KEY-----

91
tests/test_commands.py
View File

@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest):
class RunSpiderCommandTest(CommandTest):
spider_filename = 'myspider.py'
debug_log_spider = """
import scrapy
@ -507,11 +509,23 @@ class MySpider(scrapy.Spider):
return []
"""
badspider = """
import scrapy
class BadSpider(scrapy.Spider):
name = "bad"
def start_requests(self):
raise Exception("oops!")
"""
@contextmanager
def _create_file(self, content, name):
def _create_file(self, content, name=None):
tmpdir = self.mktemp()
os.mkdir(tmpdir)
fname = abspath(join(tmpdir, name))
if name:
fname = abspath(join(tmpdir, name))
else:
fname = abspath(join(tmpdir, self.spider_filename))
with open(fname, 'w') as f:
f.write(content)
try:
@ -519,12 +533,12 @@ class MySpider(scrapy.Spider):
finally:
rmtree(tmpdir)
def runspider(self, code, name='myspider.py', args=()):
def runspider(self, code, name=None, args=()):
with self._create_file(code, name) as fname:
return self.proc('runspider', fname, *args)
def get_log(self, code, name='myspider.py', args=()):
p, stdout, stderr = self.runspider(code, name=name, args=args)
def get_log(self, code, name=None, args=()):
p, stdout, stderr = self.runspider(code, name, args=args)
return stderr
def test_runspider(self):
@ -556,7 +570,7 @@ class MySpider(scrapy.Spider):
# which is intended,
# but this should not be because of DNS lookup error
# assumption: localhost will resolve in all cases (true?)
log = self.get_log("""
dnscache_spider = """
import scrapy
class MySpider(scrapy.Spider):
@ -565,23 +579,20 @@ class MySpider(scrapy.Spider):
def parse(self, response):
return {'test': 'value'}
""",
args=('-s', 'DNSCACHE_ENABLED=False'))
print(log)
"""
log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
self.assertNotIn("DNSLookupError", log)
self.assertIn("INFO: Spider opened", log)
def test_runspider_log_short_names(self):
log1 = self.get_log(self.debug_log_spider,
args=('-s', 'LOG_SHORT_NAMES=1'))
print(log1)
self.assertIn("[myspider] DEBUG: It Works!", log1)
self.assertIn("[scrapy]", log1)
self.assertNotIn("[scrapy.core.engine]", log1)
log2 = self.get_log(self.debug_log_spider,
args=('-s', 'LOG_SHORT_NAMES=0'))
print(log2)
self.assertIn("[myspider] DEBUG: It Works!", log2)
self.assertNotIn("[scrapy]", log2)
self.assertIn("[scrapy.core.engine]", log2)
@ -599,15 +610,7 @@ class MySpider(scrapy.Spider):
self.assertIn('Unable to load', log)
def test_start_requests_errors(self):
log = self.get_log("""
import scrapy
class BadSpider(scrapy.Spider):
name = "bad"
def start_requests(self):
raise Exception("oops!")
""", name="badspider.py")
print(log)
log = self.get_log(self.badspider, name='badspider.py')
self.assertIn("start_requests", log)
self.assertIn("badspider.py", log)
@ -696,6 +699,54 @@ class MySpider(scrapy.Spider):
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
spider_filename = 'myspider.pyw'
def setUp(self):
super(WindowsRunSpiderCommandTest, self).setUp()
def test_start_requests_errors(self):
log = self.get_log(self.badspider, name='badspider.pyw')
self.assertIn("start_requests", log)
self.assertIn("badspider.pyw", log)
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_run_good_spider(self):
super().test_run_good_spider()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider(self):
super().test_runspider()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_dnscache_disabled(self):
super().test_runspider_dnscache_disabled()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_log_level(self):
super().test_runspider_log_level()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_log_short_names(self):
super().test_runspider_log_short_names()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_runspider_no_spider_found(self):
super().test_runspider_no_spider_found()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_output(self):
super().test_output()
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
def test_overwrite_output(self):
super().test_overwrite_output()
def test_runspider_unable_to_load(self):
raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
class BenchCommandTest(CommandTest):
def test_run(self):

23
tests/test_downloader_handlers.py
View File

@ -877,29 +877,6 @@ class S3TestCase(unittest.TestCase):
self.assertEqual(httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
def test_request_signing5(self):
try:
import botocore # noqa: F401
except ImportError:
pass
else:
raise unittest.SkipTest(
'botocore does not support overriding date with x-amz-date')
# deletes an object from the 'johnsmith' bucket using the
# path-style and Date alternative.
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
req = Request(
's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
'Date': date,
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
# botocore does not override Date with x-amz-date
self.assertEqual(
httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.
date = 'Tue, 27 Mar 2007 21:06:08 +0000'

8
tests/test_downloadermiddleware_cookies.py
View File

@ -322,6 +322,9 @@ class CookiesMiddlewareTest(TestCase):
cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
req2 = Request('http://example.org/2', cookies=cookies2)
assert self.mw.process_request(req2, self.spider) is None
cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}]
req3 = Request('http://example.org/3', cookies=cookies3)
assert self.mw.process_request(req3, self.spider) is None
lc.check(
("scrapy.downloadermiddlewares.cookies",
"WARNING",
@ -331,6 +334,11 @@ class CookiesMiddlewareTest(TestCase):
"WARNING",
"Invalid cookie found in request <GET http://example.org/2>:"
" {'name': 'foo'} ('value' is missing)"),
("scrapy.downloadermiddlewares.cookies",
"WARNING",
"Invalid cookie found in request <GET http://example.org/3>:"
" {'name': 'foo', 'value': None} ('value' is missing)"),
)
self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')
self.assertCookieValEqual(req3.headers['Cookie'], 'key=')

7
tests/test_downloadermiddleware_httpproxy.py
View File

@ -145,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase):
req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
assert mw.process_request(req, spider) is None
self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
def test_no_proxy_invalid_values(self):
os.environ['no_proxy'] = '/var/run/docker.sock'
mw = HttpProxyMiddleware()
# '/var/run/docker.sock' may be used by the user for
# no_proxy value but is not parseable and should be skipped
assert 'no' not in mw.proxies

10
tests/test_dupefilters.py
View File

@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_crawler_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_settings_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, 'from_settings')
def test_df_direct_scheduler(self):
settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertEqual(scheduler.df.method, 'n/a')
@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)

252
tests/test_feedexport.py
View File

@ -13,7 +13,7 @@ from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
from unittest import mock
from urllib.parse import urljoin, urlparse, quote
from urllib.parse import urljoin, quote
from urllib.request import pathname2url
import lxml.etree
@ -41,10 +41,9 @@ from scrapy.extensions.feedexport import (
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import (
assert_aws_environ,
get_s3_content_and_delete,
get_crawler,
mock_google_cloud_storage,
skip_if_no_boto,
)
from tests.mockserver import MockFTPServer, MockServer
@ -227,10 +226,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
class S3FeedStorageTest(unittest.TestCase):
def test_parse_credentials(self):
try:
import botocore # noqa: F401
except ImportError:
raise unittest.SkipTest("S3FeedStorage requires botocore")
skip_if_no_boto()
aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
crawler = get_crawler(settings_dict=aws_credentials)
@ -256,21 +252,42 @@ class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store(self):
assert_aws_environ()
uri = os.environ.get('S3_TEST_FILE_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
storage = S3FeedStorage(uri, access_key, secret_key)
skip_if_no_boto()
settings = {
'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key',
}
crawler = get_crawler(settings_dict=settings)
bucket = 'mybucket'
key = 'export.csv'
storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
verifyObject(IFeedStorage, storage)
file = storage.open(scrapy.Spider("default"))
expected_content = b"content: \xe2\x98\x83"
file.write(expected_content)
yield storage.store(file)
u = urlparse(uri)
content = get_s3_content_and_delete(u.hostname, u.path[1:])
self.assertEqual(content, expected_content)
file = mock.MagicMock()
from botocore.stub import Stubber
with Stubber(storage.s3_client) as stub:
stub.add_response(
'put_object',
expected_params={
'Body': file,
'Bucket': bucket,
'Key': key,
},
service_response={},
)
yield storage.store(file)
stub.assert_no_pending_responses()
self.assertEqual(
file.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
mock.call.close(),
]
)
def test_init_without_acl(self):
storage = S3FeedStorage(
@ -324,11 +341,7 @@ class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store_botocore_without_acl(self):
try:
import botocore # noqa: F401
except ImportError:
raise unittest.SkipTest('botocore is required')
skip_if_no_boto()
storage = S3FeedStorage(
's3://mybucket/export.csv',
'access_key',
@ -344,11 +357,7 @@ class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store_botocore_with_acl(self):
try:
import botocore # noqa: F401
except ImportError:
raise unittest.SkipTest('botocore is required')
skip_if_no_boto()
storage = S3FeedStorage(
's3://mybucket/export.csv',
'access_key',
@ -366,57 +375,6 @@ class S3FeedStorageTest(unittest.TestCase):
'custom-acl'
)
@defer.inlineCallbacks
def test_store_not_botocore_without_acl(self):
storage = S3FeedStorage(
's3://mybucket/export.csv',
'access_key',
'secret_key',
)
self.assertEqual(storage.access_key, 'access_key')
self.assertEqual(storage.secret_key, 'secret_key')
self.assertEqual(storage.acl, None)
storage.is_botocore = False
storage.connect_s3 = mock.MagicMock()
self.assertFalse(storage.is_botocore)
yield storage.store(BytesIO(b'test file'))
conn = storage.connect_s3(*storage.connect_s3.call_args)
bucket = conn.get_bucket(*conn.get_bucket.call_args)
key = bucket.new_key(*bucket.new_key.call_args)
self.assertNotIn(
dict(policy='custom-acl'),
key.set_contents_from_file.call_args
)
@defer.inlineCallbacks
def test_store_not_botocore_with_acl(self):
storage = S3FeedStorage(
's3://mybucket/export.csv',
'access_key',
'secret_key',
'custom-acl'
)
self.assertEqual(storage.access_key, 'access_key')
self.assertEqual(storage.secret_key, 'secret_key')
self.assertEqual(storage.acl, 'custom-acl')
storage.is_botocore = False
storage.connect_s3 = mock.MagicMock()
self.assertFalse(storage.is_botocore)
yield storage.store(BytesIO(b'test file'))
conn = storage.connect_s3(*storage.connect_s3.call_args)
bucket = conn.get_bucket(*conn.get_bucket.call_args)
key = bucket.new_key(*bucket.new_key.call_args)
self.assertIn(
dict(policy='custom-acl'),
key.set_contents_from_file.call_args
)
def test_overwrite_default(self):
with LogCapture() as log:
S3FeedStorage(
@ -1252,6 +1210,43 @@ class FeedExportTest(FeedExportTestBase):
for fmt in ['json', 'xml', 'csv']:
self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
@defer.inlineCallbacks
def test_extend_kwargs(self):
items = [{'foo': 'FOO', 'bar': 'BAR'}]
expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
test_cases = [
# with title
{
'options': {
'format': 'csv',
'item_export_kwargs': {'include_headers_line': True},
},
'expected': expected_with_title_csv,
},
# without title
{
'options': {
'format': 'csv',
'item_export_kwargs': {'include_headers_line': False},
},
'expected': expected_without_title_csv,
},
]
for row in test_cases:
feed_options = row['options']
settings = {
'FEEDS': {
self._random_temp_filename(): feed_options,
},
'FEED_EXPORT_INDENT': None,
}
data = yield self.exported_data(items, settings)
self.assertEqual(row['expected'], data[feed_options['format']])
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True
@ -1557,46 +1552,53 @@ class BatchDeliveriesTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_s3_export(self):
"""
Test export of items into s3 bucket.
S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
to perform this test:
[testenv]
setenv =
AWS_SECRET_ACCESS_KEY = ABCD
AWS_ACCESS_KEY_ID = EFGH
S3_TEST_BUCKET_NAME = IJKL
"""
try:
import boto3
except ImportError:
raise unittest.SkipTest("S3FeedStorage requires boto3")
skip_if_no_boto()
assert_aws_environ()
s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if not s3_test_bucket_name:
raise unittest.SkipTest("No S3 BUCKET available for testing")
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
prefix = f'tmp/{filename}'
s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json'
storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
settings = Settings({
'FEEDS': {
s3_test_file_uri: {
'format': 'json',
},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
})
bucket = 'mybucket'
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
]
class CustomS3FeedStorage(S3FeedStorage):
stubs = []
def open(self, *args, **kwargs):
from botocore.stub import ANY, Stubber
stub = Stubber(self.s3_client)
stub.activate()
CustomS3FeedStorage.stubs.append(stub)
stub.add_response(
'put_object',
expected_params={
'Body': ANY,
'Bucket': bucket,
'Key': ANY,
},
service_response={},
)
return super().open(*args, **kwargs)
key = 'export.csv'
uri = f's3://{bucket}/{key}/%(batch_time)s.json'
batch_item_count = 1
settings = {
'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key',
'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
'FEED_STORAGES': {
's3': CustomS3FeedStorage,
},
'FEEDS': {
uri: {
'format': 'json',
},
},
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, uri)
verifyObject(IFeedStorage, storage)
class TestSpider(scrapy.Spider):
@ -1606,22 +1608,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
for item in items:
yield item
s3 = boto3.resource('s3')
my_bucket = s3.Bucket(s3_test_bucket_name)
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
with MockServer() as s:
with MockServer() as server:
runner = CrawlerRunner(Settings(settings))
TestSpider.start_urls = [s.url('/')]
TestSpider.start_urls = [server.url('/')]
yield runner.crawl(TestSpider)
for file_uri in my_bucket.objects.filter(Prefix=prefix):
content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
if not content and not items:
break
content = json.loads(content.decode('utf-8'))
expected_batch, items = items[:batch_size], items[batch_size:]
self.assertEqual(expected_batch, content)
self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
for stub in CustomS3FeedStorage.stubs[:-1]:
stub.assert_no_pending_responses()
class FeedExportInitTest(unittest.TestCase):

4
tests/test_logformatter.py
View File

@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
self.base_settings = {
'LOG_LEVEL': 'DEBUG',
'ITEM_PIPELINES': {
__name__ + '.DropSomeItemsPipeline': 300,
DropSomeItemsPipeline: 300,
},
}
@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
@defer.inlineCallbacks
def test_skip_messages(self):
settings = self.base_settings.copy()
settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)

105
tests/test_pipeline_files.py
View File

@ -1,6 +1,7 @@
import os
import random
import time
from datetime import datetime
from io import BytesIO
from shutil import rmtree
from tempfile import mkdtemp
@ -22,13 +23,11 @@ from scrapy.pipelines.files import (
S3FilesStore,
)
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore
from scrapy.utils.test import (
assert_aws_environ,
assert_gcs_environ,
get_ftp_content_and_delete,
get_gcs_content_and_delete,
get_s3_content_and_delete,
skip_if_no_boto,
)
@ -415,38 +414,88 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
assert_aws_environ()
uri = os.environ.get('S3_TEST_FILE_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
data = b"TestS3FilesStore: \xe2\x98\x83"
buf = BytesIO(data)
skip_if_no_boto()
bucket = 'mybucket'
key = 'export.csv'
uri = f's3://{bucket}/{key}'
buffer = mock.MagicMock()
meta = {'foo': 'bar'}
path = ''
content_type = 'image/png'
store = S3FilesStore(uri)
yield store.persist_file(
path, buf, info=None, meta=meta,
headers={'Content-Type': 'image/png'})
s = yield store.stat_file(path, info=None)
self.assertIn('last_modified', s)
self.assertIn('checksum', s)
self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
u = urlparse(uri)
content, key = get_s3_content_and_delete(
u.hostname, u.path[1:], with_key=True)
self.assertEqual(content, data)
if is_botocore():
self.assertEqual(key['Metadata'], {'foo': 'bar'})
from botocore.stub import Stubber
with Stubber(store.s3_client) as stub:
stub.add_response(
'put_object',
expected_params={
'ACL': S3FilesStore.POLICY,
'Body': buffer,
'Bucket': bucket,
'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
'ContentType': content_type,
'Key': key,
'Metadata': meta,
},
service_response={},
)
yield store.persist_file(
path,
buffer,
info=None,
meta=meta,
headers={'Content-Type': content_type},
)
stub.assert_no_pending_responses()
self.assertEqual(
key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
self.assertEqual(key['ContentType'], 'image/png')
else:
self.assertEqual(key.metadata, {'foo': 'bar'})
buffer.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
]
)
@defer.inlineCallbacks
def test_stat(self):
skip_if_no_boto()
bucket = 'mybucket'
key = 'export.csv'
uri = f's3://{bucket}/{key}'
checksum = '3187896a9657a28163abb31667df64c8'
last_modified = datetime(2019, 12, 1)
store = S3FilesStore(uri)
from botocore.stub import Stubber
with Stubber(store.s3_client) as stub:
stub.add_response(
'head_object',
expected_params={
'Bucket': bucket,
'Key': key,
},
service_response={
'ETag': f'"{checksum}"',
'LastModified': last_modified,
},
)
file_stats = yield store.stat_file('', info=None)
self.assertEqual(
key.cache_control, S3FilesStore.HEADERS['Cache-Control'])
self.assertEqual(key.content_type, 'image/png')
file_stats,
{
'checksum': checksum,
'last_modified': last_modified.timestamp(),
},
)
stub.assert_no_pending_responses()
class TestGCSFilesStore(unittest.TestCase):

2
tests/test_pipelines.py
View File

@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase):
def _create_crawler(self, pipeline_class):
settings = {
'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
'ITEM_PIPELINES': {pipeline_class: 1},
}
crawler = get_crawler(ItemSpider, settings)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)

14
tests/test_request_attribute_binding.py
View File

@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
"DOWNLOADER_MIDDLEWARES": {
__name__ + ".RaiseExceptionRequestMiddleware": 590,
RaiseExceptionRequestMiddleware: 590,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
@ -119,7 +119,7 @@ class CrawlTestCase(TestCase):
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
"DOWNLOADER_MIDDLEWARES": {
__name__ + ".ProcessResponseMiddleware": 595,
ProcessResponseMiddleware: 595,
}
})
crawler = runner.create_crawler(SingleRequestSpider)
@ -149,8 +149,8 @@ class CrawlTestCase(TestCase):
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
"DOWNLOADER_MIDDLEWARES": {
__name__ + ".RaiseExceptionRequestMiddleware": 590,
__name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
RaiseExceptionRequestMiddleware: 590,
CatchExceptionOverrideRequestMiddleware: 595,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
@ -170,8 +170,8 @@ class CrawlTestCase(TestCase):
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
"DOWNLOADER_MIDDLEWARES": {
__name__ + ".RaiseExceptionRequestMiddleware": 590,
__name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
RaiseExceptionRequestMiddleware: 590,
CatchExceptionDoNotOverrideRequestMiddleware: 595,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
@ -188,7 +188,7 @@ class CrawlTestCase(TestCase):
"""
runner = CrawlerRunner(settings={
"DOWNLOADER_MIDDLEWARES": {
__name__ + ".AlternativeCallbacksMiddleware": 595,
AlternativeCallbacksMiddleware: 595,
}
})
crawler = runner.create_crawler(AlternativeCallbacksSpider)

4
tests/test_request_cb_kwargs.py
View File

@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider):
name = 'kwargs'
custom_settings = {
'DOWNLOADER_MIDDLEWARES': {
__name__ + '.InjectArgumentsDownloaderMiddleware': 750,
InjectArgumentsDownloaderMiddleware: 750,
},
'SPIDER_MIDDLEWARES': {
__name__ + '.InjectArgumentsSpiderMiddleware': 750,
InjectArgumentsSpiderMiddleware: 750,
},
}

45
tests/test_spider.py
View File

@ -1,8 +1,8 @@
import gzip
import inspect
from unittest import mock
import warnings
from io import BytesIO
from unittest import mock
from testfixtures import LogCapture
from twisted.trial import unittest
@ -20,7 +20,6 @@ from scrapy.spiders import (
XMLFeedSpider,
)
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.test import get_crawler
@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest):
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
def process_request_change_domain(request):
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace('.org', '.com'))
class _CrawlSpider(self.spider_class):
@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest):
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
with warnings.catch_warnings(record=True) as cw:
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.com/somepage/item/12.html',
'http://example.com/about.html',
'http://example.com/nofollow.html'])
self.assertEqual(len(cw), 1)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.com/somepage/item/12.html',
'http://example.com/about.html',
'http://example.com/nofollow.html'])
def test_process_request_with_response(self):
@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest):
Rule(LinkExtractor(), process_request='process_request_upper'),
)
def process_request_upper(self, request):
def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())
with warnings.catch_warnings(record=True) as cw:
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
self.assertEqual(len(cw), 1)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
def test_process_request_instance_method_with_response(self):

92
tests/test_spidermiddleware_output_chain.py
View File

@ -16,11 +16,20 @@ class LogExceptionMiddleware:
# ================================================================================
# (0) recover from an exception on a spider callback
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
return [
{'from': 'process_spider_exception'},
Request(response.url, meta={'dont_fail': True}, dont_filter=True),
]
class RecoverySpider(Spider):
name = 'RecoverySpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.RecoveryMiddleware': 10,
RecoveryMiddleware: 10,
},
}
@ -34,15 +43,6 @@ class RecoverySpider(Spider):
raise TabError()
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
return [
{'from': 'process_spider_exception'},
Request(response.url, meta={'dont_fail': True}, dont_filter=True),
]
# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware:
@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
custom_settings = {
'SPIDER_MIDDLEWARES': {
# spider
__name__ + '.LogExceptionMiddleware': 10,
__name__ + '.FailProcessSpiderInputMiddleware': 8,
__name__ + '.LogExceptionMiddleware': 6,
FailProcessSpiderInputMiddleware: 8,
LogExceptionMiddleware: 6,
# engine
}
}
@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider):
name = 'GeneratorCallbackSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.LogExceptionMiddleware': 10,
LogExceptionMiddleware: 10,
},
}
@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.LogExceptionMiddleware': 100000,
LogExceptionMiddleware: 100000,
},
}
@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider):
name = 'NotGeneratorCallbackSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.LogExceptionMiddleware': 10,
LogExceptionMiddleware: 10,
},
}
@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.LogExceptionMiddleware': 100000,
LogExceptionMiddleware: 100000,
},
}
# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
class GeneratorOutputChainSpider(Spider):
name = 'GeneratorOutputChainSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.GeneratorFailMiddleware': 10,
__name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,
__name__ + '.GeneratorRecoverMiddleware': 5,
__name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,
},
}
def start_requests(self):
yield Request(self.mockserver.url('/status?n=200'))
def parse(self, response):
yield {'processed': ['parse-first-item']}
yield {'processed': ['parse-second-item']}
class _GeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
# ================================================================================
# (5) exceptions from a middleware process_spider_output method (not generator)
class NotGeneratorOutputChainSpider(Spider):
name = 'NotGeneratorOutputChainSpider'
class GeneratorOutputChainSpider(Spider):
name = 'GeneratorOutputChainSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
__name__ + '.NotGeneratorFailMiddleware': 10,
__name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,
__name__ + '.NotGeneratorRecoverMiddleware': 5,
__name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,
GeneratorFailMiddleware: 10,
GeneratorDoNothingAfterFailureMiddleware: 8,
GeneratorRecoverMiddleware: 5,
GeneratorDoNothingAfterRecoveryMiddleware: 3,
},
}
def start_requests(self):
return [Request(self.mockserver.url('/status?n=200'))]
yield Request(self.mockserver.url('/status?n=200'))
def parse(self, response):
return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
yield {'processed': ['parse-first-item']}
yield {'processed': ['parse-second-item']}
# ================================================================================
# (5) exceptions from a middleware process_spider_output method (not generator)
class _NotGeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
out = []
@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
pass
class NotGeneratorOutputChainSpider(Spider):
name = 'NotGeneratorOutputChainSpider'
custom_settings = {
'SPIDER_MIDDLEWARES': {
NotGeneratorFailMiddleware: 10,
NotGeneratorDoNothingAfterFailureMiddleware: 8,
NotGeneratorRecoverMiddleware: 5,
NotGeneratorDoNothingAfterRecoveryMiddleware: 3,
},
}
def start_requests(self):
return [Request(self.mockserver.url('/status?n=200'))]
def parse(self, response):
return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
# ================================================================================
class TestSpiderMiddleware(TestCase):
@classmethod

2
tests/test_utils_conf.py
View File

@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"store_empty": True,
"uri_params": (1, 2, 3, 4),
"batch_item_count": 2,
"item_export_kwargs": dict(),
})
def test_feed_complete_default_values_from_settings_non_empty(self):
@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"store_empty": True,
"uri_params": None,
"batch_item_count": 2,
"item_export_kwargs": dict(),
})

55
tests/test_utils_iterators.py
View File

@ -1,5 +1,6 @@
import os
from pytest import mark
from twisted.trial import unittest
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@ -134,7 +135,6 @@ class XmliterTestCase(unittest.TestCase):
"""
response = XmlResponse(url='http://mydummycompany.com', body=body)
my_iter = self.xmliter(response, 'item')
node = next(my_iter)
node.register_namespace('g', 'http://base.google.com/ns/1.0')
self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
@ -150,6 +150,55 @@ class XmliterTestCase(unittest.TestCase):
self.assertEqual(node.xpath('id/text()').getall(), [])
self.assertEqual(node.xpath('price/text()').getall(), [])
def test_xmliter_namespaced_nodename(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
<channel>
<title>My Dummy Company</title>
<link>http://www.mydummycompany.com</link>
<description>This is a dummy company. We do nothing.</description>
<item>
<title>Item 1</title>
<description>This is item 1</description>
<link>http://www.mydummycompany.com/items/1</link>
<g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
<g:id>ITEM_1</g:id>
<g:price>400</g:price>
</item>
</channel>
</rss>
"""
response = XmlResponse(url='http://mydummycompany.com', body=body)
my_iter = self.xmliter(response, 'g:image_link')
node = next(my_iter)
node.register_namespace('g', 'http://base.google.com/ns/1.0')
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
def test_xmliter_namespaced_nodename_missing(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
<channel>
<title>My Dummy Company</title>
<link>http://www.mydummycompany.com</link>
<description>This is a dummy company. We do nothing.</description>
<item>
<title>Item 1</title>
<description>This is item 1</description>
<link>http://www.mydummycompany.com/items/1</link>
<g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
<g:id>ITEM_1</g:id>
<g:price>400</g:price>
</item>
</channel>
</rss>
"""
response = XmlResponse(url='http://mydummycompany.com', body=body)
my_iter = self.xmliter(response, 'g:link_image')
with self.assertRaises(StopIteration):
next(my_iter)
def test_xmliter_exception(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -183,6 +232,10 @@ class XmliterTestCase(unittest.TestCase):
class LxmlXmliterTestCase(XmliterTestCase):
xmliter = staticmethod(xmliter_lxml)
@mark.xfail(reason='known bug of the current implementation')
def test_xmliter_namespaced_nodename(self):
super().test_xmliter_namespaced_nodename()
def test_xmliter_iterate_namespace(self):
body = b"""
<?xml version="1.0" encoding="UTF-8"?>

3
tox.ini
View File

@ -12,10 +12,9 @@ deps =
-ctests/constraints.txt
-rtests/requirements-py3.txt
# Extras
Twisted[http2]>=17.9.0
boto3>=1.13.0
botocore>=1.4.87
Pillow>=4.0.0
Twisted[http2]>=17.9.0
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID