RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into request-types

This commit is contained in:
Eugenio Lacuesta 2021-06-01 11:01:23 -03:00
parent 19c7415aae 23cfdb058e
commit c92e8ad0a5
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
25 changed files with 315 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.flake8 Normal file
View File

@ -0,0 +1,19 @@
[flake8]
max-line-length = 119
ignore = W503
exclude =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

2
.gitignore vendored
View File

@ -14,6 +14,8 @@ htmlcov/
.coverage
.pytest_cache/
.coverage.*
coverage.*
test-output.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt

18
conftest.py
View File

@ -1,6 +1,7 @@
from pathlib import Path
import pytest
from twisted.web.http import H2_ENABLED
from scrapy.utils.reactor import install_reactor
@ -20,10 +21,19 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
with open('tests/ignores.txt') as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
(
'scrapy/core/downloader/handlers/http2.py',
*_py_files("scrapy/core/http2"),
)
)
@pytest.fixture()

3
docs/topics/feed-exports.rst
View File

@ -135,6 +135,9 @@ Here are some examples to illustrate:
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
.. _topics-feed-storage-backends:

17
docs/topics/request-response.rst
View File

@ -26,10 +26,6 @@ Request objects
.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
@ -205,6 +201,8 @@ Request objects
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. autoattribute:: Request.attributes
.. method:: Request.copy()
Return a new Request which is a copy of this Request. See also:
@ -220,6 +218,15 @@ Request objects
.. automethod:: from_curl
.. automethod:: to_dict
Other functions related to requests
-----------------------------------
.. autofunction:: scrapy.utils.request.request_from_dict
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@ -642,6 +649,8 @@ dealing with JSON requests.
data into JSON format.
:type dumps_kwargs: dict
.. autoattribute:: JsonRequest.attributes
JsonRequest usage example
-------------------------

17
docs/topics/settings.rst
View File

@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::
.. _http2:
The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
:setting:`DOWNLOAD_HANDLERS` as follows::
The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
}
#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to
enable HTTP/2 support in Twisted.
#. Update :setting:`DOWNLOAD_HANDLERS` as follows::
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
}
.. warning::
@ -1619,7 +1623,7 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
.. setting:: USER_AGENT
@ -1642,7 +1646,6 @@ case to see how to enable and use them.
.. settingslist::
.. _Amazon web services: https://aws.amazon.com/
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search

8
docs/topics/spiders.rst
View File

@ -294,6 +294,14 @@ The above example can also be written as follows::
def start_requests(self):
yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
specify spider arguments when calling
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
process = CrawlerProcess()
process.crawl(MySpider, category="electronics")
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,

19
pytest.ini
View File

@ -20,20 +20,5 @@ addopts =
--ignore=docs/utils
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-max-line-length = 119
flake8-ignore =
W503
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
filterwarnings=
ignore::DeprecationWarning:twisted.web.test.test_webclient

7
scrapy/core/downloader/handlers/http11.py
View File

@ -98,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
with this endpoint comes from the pool and a CONNECT has already been issued
for it.
"""
_responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
_truncatedLength = 1000
_responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
@ -144,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
else:
extra = rcvd_bytes[:32]
extra = rcvd_bytes[:self._truncatedLength]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy '
f'{self._host}:{self._port} [{extra!r}]')

62
scrapy/http/request/__init__.py
View File

@ -4,10 +4,12 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
from typing import Callable, List, Optional, Type, TypeVar, Union
import inspect
from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
from w3lib.url import safe_url_string
import scrapy
from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
from scrapy.utils.curl import curl_to_request_kwargs
@ -20,6 +22,23 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
class Request(object_ref):
"""Represents an HTTP request, which is usually generated in a Spider and
executed by the Downloader, thus generating a :class:`Response`.
"""
attributes: Tuple[str, ...] = (
"url", "callback", "method", "headers", "body",
"cookies", "meta", "encoding", "priority",
"dont_filter", "errback", "flags", "cb_kwargs",
)
"""A tuple of :class:`str` objects containing the name of all public
attributes of the class that are also keyword parameters of the
``__init__`` method.
Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and
:func:`~scrapy.utils.request.request_from_dict`.
"""
def __init__(
self,
url: str,
@ -27,7 +46,7 @@ class Request(object_ref):
method: str = "GET",
headers: Optional[dict] = None,
body: Optional[Union[bytes, str]] = None,
cookies: Optional[Union[dict, List[dict]]]=None,
cookies: Optional[Union[dict, List[dict]]] = None,
meta: Optional[dict] = None,
encoding: str = "utf-8",
priority: int = 0,
@ -112,8 +131,7 @@ class Request(object_ref):
def replace(self, *args, **kwargs) -> RequestTypeVar:
"""Create a new Request with the same attributes except for those given new values"""
for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
@ -152,3 +170,39 @@ class Request(object_ref):
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
request_kwargs.update(kwargs)
return cls(**request_kwargs)
def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
"""Return a dictionary containing the Request's data.
Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
If a spider is given, this method will try to find out the name of the spider methods used as callback
and errback and include them in the output dict, raising an exception if they cannot be found.
"""
d = {
"url": self.url, # urls are safe (safe_string_url)
"callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
"errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
"headers": dict(self.headers),
}
for attr in self.attributes:
d.setdefault(attr, getattr(self, attr))
if type(self) is not Request:
d["_class"] = self.__module__ + '.' + self.__class__.__name__
return d
def _find_method(obj, func):
"""Helper function for Request.to_dict"""
# Only instance methods contain ``__func__``
if obj and hasattr(func, '__func__'):
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original function object because instance
# method objects are generated each time attribute is retrieved from instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError(f"Function {func} is not an instance method in: {obj}")

8
scrapy/http/request/json_request.py
View File

@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst
import copy
import json
import warnings
from typing import Tuple
from scrapy.http.request import Request
from scrapy.utils.deprecate import create_deprecated_class
class JsonRequest(Request):
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
def __init__(self, *args, **kwargs):
dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
dumps_kwargs.setdefault('sort_keys', True)
@ -36,6 +40,10 @@ class JsonRequest(Request):
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@property
def dumps_kwargs(self):
return self._dumps_kwargs
def replace(self, *args, **kwargs):
body_passed = kwargs.get('body', None) is not None
data = kwargs.pop('data', None)

8
scrapy/squeues.py
View File

@ -9,7 +9,7 @@ import pickle
from queuelib import queue
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy.utils.request import request_from_dict
def _with_mkdir(queue_class):
@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class):
return cls(crawler, key)
def push(self, request):
request = request_to_dict(request, self.spider)
request = request.to_dict(spider=self.spider)
return super().push(request)
def pop(self):
request = super().pop()
if not request:
return None
return request_from_dict(request, self.spider)
return request_from_dict(request, spider=self.spider)
def peek(self):
"""Returns the next object to be returned by :meth:`pop`,
@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class):
request = super().peek()
if not request:
return None
return request_from_dict(request, self.spider)
return request_from_dict(request, spider=self.spider)
return ScrapyRequestQueue

103
scrapy/utils/reqser.py
View File

@ -1,95 +1,22 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
import inspect
import warnings
from typing import Optional
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.request import request_from_dict as _from_dict
def request_to_dict(request, spider=None):
"""Convert Request object to a dict.
If a spider is given, it will try to find out the name of the spider method
used in the callback and store that as the callback.
"""
cb = request.callback
if callable(cb):
cb = _find_method(spider, cb)
eb = request.errback
if callable(eb):
eb = _find_method(spider, eb)
d = {
'url': to_unicode(request.url), # urls should be safe (safe_string_url)
'callback': cb,
'errback': eb,
'method': request.method,
'headers': dict(request.headers),
'body': request.body,
'cookies': request.cookies,
'meta': request.meta,
'_encoding': request._encoding,
'priority': request.priority,
'dont_filter': request.dont_filter,
'flags': request.flags,
'cb_kwargs': request.cb_kwargs,
}
if type(request) is not Request:
d['_class'] = request.__module__ + '.' + request.__class__.__name__
return d
warnings.warn(
("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
" and/or scrapy.utils.request.request_from_dict instead"),
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def request_from_dict(d, spider=None):
"""Create Request object from a dict.
If a spider is given, it will try to resolve the callbacks looking at the
spider for methods with the same name.
"""
cb = d['callback']
if cb and spider:
cb = _get_method(spider, cb)
eb = d['errback']
if eb and spider:
eb = _get_method(spider, eb)
request_cls = load_object(d['_class']) if '_class' in d else Request
return request_cls(
url=to_unicode(d['url']),
callback=cb,
errback=eb,
method=d['method'],
headers=d['headers'],
body=d['body'],
cookies=d['cookies'],
meta=d['meta'],
encoding=d['_encoding'],
priority=d['priority'],
dont_filter=d['dont_filter'],
flags=d.get('flags'),
cb_kwargs=d.get('cb_kwargs'),
)
def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:
return request.to_dict(spider=spider)
def _find_method(obj, func):
# Only instance methods contain ``__func__``
if obj and hasattr(func, '__func__'):
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original
# function object because instance method objects
# are generated each time attribute is retrieved from
# instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError(f"Function {func} is not an instance method in: {obj}")
def _get_method(obj, name):
name = str(name)
try:
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")
def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":
return _from_dict(d, spider=spider)

27
scrapy/utils/request.py
View File

@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy.http import Request
from scrapy import Request, Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]:
if referrer is None:
return referrer
return to_unicode(referrer, errors='replace')
def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
"""Create a :class:`~scrapy.Request` object from a dict.
If a spider is given, it will try to resolve the callbacks looking at the
spider for methods with the same name.
"""
request_cls = load_object(d["_class"]) if "_class" in d else Request
kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
if d.get("callback") and spider:
kwargs["callback"] = _get_method(spider, d["callback"])
if d.get("errback") and spider:
kwargs["errback"] = _get_method(spider, d["errback"])
return request_cls(**kwargs)
def _get_method(obj, name):
"""Helper function for request_from_dict"""
name = str(name)
try:
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")

3
setup.py
View File

@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
install_requires = [
'Twisted[http2]>=17.9.0',
'Twisted>=17.9.0',
'cryptography>=2.0',
'cssselect>=0.9.1',
'itemloaders>=1.0.1',
@ -31,7 +31,6 @@ install_requires = [
'zope.interface>=4.1.3',
'protego>=0.1.15',
'itemadapter>=0.1.0',
'h2>=3.0,<4.0',
'setuptools',
]
extras_require = {}

26
tests/test_downloader_handlers_http2.py
View File

@ -1,5 +1,5 @@
import json
from unittest import mock
from unittest import mock, skipIf
from pytest import mark
from testfixtures import LogCapture
@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor
from twisted.trial import unittest
from twisted.web import server
from twisted.web.error import SchemeNotSupported
from twisted.web.http import H2_ENABLED
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import create_instance
@ -21,11 +21,17 @@ from tests.test_downloader_handlers import (
)
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2TestCase(Https11TestCase):
scheme = 'https'
download_handler_cls = H2DownloadHandler
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
@classmethod
def setUpClass(cls):
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
cls.download_handler_cls = H2DownloadHandler
def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
d = self.download_request(request, Spider("foo"))
@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase):
super(Https2InvalidDNSPattern, self).setUp()
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2CustomCiphers(Https11CustomCiphers):
scheme = 'https'
download_handler_cls = H2DownloadHandler
@classmethod
def setUpClass(cls):
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
cls.download_handler_cls = H2DownloadHandler
class Http2MockServerTestCase(Http11MockServerTestCase):
@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
}
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ProxyTestCase(Http11ProxyTestCase):
# only used for HTTPS tests
keyfile = 'keys/localhost.key'
@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
scheme = 'https'
host = u'127.0.0.1'
download_handler_cls = H2DownloadHandler
expected_http_proxy_request_body = b'/'
@classmethod
def setUpClass(cls):
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
cls.download_handler_cls = H2DownloadHandler
def setUp(self):
site = server.Site(UriResource(), timeout=None)
self.port = reactor.listenSSL(

12
tests/test_exporters.py
View File

@ -6,12 +6,14 @@ import tempfile
import unittest
from io import BytesIO
from datetime import datetime
from warnings import catch_warnings, filterwarnings
import lxml.etree
from itemadapter import ItemAdapter
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exporters import (
BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,
@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_binary(self):
exporter = PythonItemExporter(binary=True)
value = self.item_class(name='John\xa3', age='22')
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
self.assertEqual(expected, exporter.export_item(value))
with catch_warnings():
filterwarnings('ignore', category=ScrapyDeprecationWarning)
exporter = PythonItemExporter(binary=True)
value = self.item_class(name='John\xa3', age='22')
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
self.assertEqual(expected, exporter.export_item(value))
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()

4
tests/test_feedexport.py
View File

@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
class DummyBlockingFeedStorage(BlockingFeedStorage):
def __init__(self, uri):
def __init__(self, uri, *args, feed_options=None):
self.path = file_uri_to_path(uri)
def _store_in_thread(self, file):
@ -541,7 +541,7 @@ class LogOnStoreFileStorage:
It can be used to make sure `store` method is invoked.
"""
def __init__(self, uri):
def __init__(self, uri, feed_options=None):
self.path = file_uri_to_path(uri)
self.logger = getLogger()

13
tests/test_http2_client_protocol.py
View File

@ -5,10 +5,9 @@ import re
import shutil
import string
from ipaddress import IPv4Address
from unittest import mock
from unittest import mock, skipIf
from urllib.parse import urlencode
from h2.exceptions import InvalidBodyLengthError
from twisted.internet import reactor
from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
from twisted.web.client import ResponseFailed, URI
from twisted.web.http import Request as TxRequest
from twisted.web.http import H2_ENABLED, Request as TxRequest
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.static import File
from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
from scrapy.http import Request, Response, JsonRequest
from scrapy.settings import Settings
from scrapy.spiders import Spider
@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
return PrivateCertificate.loadPEM(pem)
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ClientProtocolTestCase(TestCase):
scheme = 'https'
key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase):
uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
self.conn_closed_deferred = Deferred()
from scrapy.core.http2.protocol import H2ClientFactory
h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
self.client = yield client_endpoint.connect(h2_client_factory)
@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_failure(failure: Failure):
self.assertTrue(len(failure.value.reasons) > 0)
from h2.exceptions import InvalidBodyLengthError
self.assertTrue(any(
isinstance(error, InvalidBodyLengthError)
for error in failure.value.reasons
@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_inactive_stream(failure):
self.assertIsNotNone(failure.check(ResponseFailed))
from scrapy.core.http2.stream import InactiveStreamClosed
self.assertTrue(any(
isinstance(e, InactiveStreamClosed)
for e in failure.value.reasons
@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase):
request = Request(url)
def assert_invalid_hostname(failure: Failure):
from scrapy.core.http2.stream import InvalidHostname
self.assertIsNotNone(failure.check(InvalidHostname))
error_msg = str(failure.value)
self.assertIn('localhost', error_msg)
@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase):
def assert_timeout_error(failure: Failure):
for err in failure.value.reasons:
from scrapy.core.http2.protocol import H2ClientProtocol
if isinstance(err, TimeoutError):
self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
break

12
tests/test_http_response.py
View File

@ -1,6 +1,6 @@
import unittest
from unittest import mock
from warnings import catch_warnings
from warnings import catch_warnings, filterwarnings
from w3lib.encoding import resolve_encoding
@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase):
assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
self.assertEqual(response.body_as_unicode(), body_unicode)
with catch_warnings():
filterwarnings("ignore", category=ScrapyDeprecationWarning)
self.assertEqual(response.body_as_unicode(), body_unicode)
self.assertEqual(response.text, body_unicode)
def _assert_response_encoding(self, response, encoding):
@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest):
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
# check body_as_unicode
self.assertTrue(isinstance(r1.body_as_unicode(), str))
self.assertEqual(r1.body_as_unicode(), unicode_string)
with catch_warnings():
filterwarnings("ignore", category=ScrapyDeprecationWarning)
self.assertTrue(isinstance(r1.body_as_unicode(), str))
self.assertEqual(r1.body_as_unicode(), unicode_string)
# check response.text
self.assertTrue(isinstance(r1.text, str))

22
tests/test_item.py
View File

@ -1,6 +1,6 @@
import unittest
from unittest import mock
from warnings import catch_warnings
from warnings import catch_warnings, filterwarnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase):
class SubclassedItem(Item):
pass
self.assertTrue(isinstance(BaseItem(), BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
self.assertTrue(isinstance(Item(), BaseItem))
self.assertTrue(isinstance(SubclassedItem(), BaseItem))
with catch_warnings():
filterwarnings("ignore", category=ScrapyDeprecationWarning)
self.assertTrue(isinstance(BaseItem(), BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
self.assertTrue(isinstance(Item(), BaseItem))
self.assertTrue(isinstance(SubclassedItem(), BaseItem))
# make sure internal checks using private _BaseItem class succeed
self.assertTrue(isinstance(BaseItem(), _BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
self.assertTrue(isinstance(Item(), _BaseItem))
self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
# make sure internal checks using private _BaseItem class succeed
self.assertTrue(isinstance(BaseItem(), _BaseItem))
self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
self.assertTrue(isinstance(Item(), _BaseItem))
self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
def test_deprecation_warning(self):
"""

76
tests/test_utils_reqser.py → tests/test_request_dict.py
View File

@ -1,8 +1,16 @@
import sys
import unittest
import warnings
from contextlib import suppress
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy import Spider, Request
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import FormRequest, JsonRequest
from scrapy.utils.request import request_from_dict
class CustomRequest(Request):
pass
class RequestSerializationTest(unittest.TestCase):
@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase):
priority=20,
meta={'a': 'b'},
cb_kwargs={'k': 'v'},
flags=['testFlag'])
flags=['testFlag'],
)
self._assert_serializes_ok(r, spider=self.spider)
def test_latin1_body(self):
@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r)
def _assert_serializes_ok(self, request, spider=None):
d = request_to_dict(request, spider=spider)
d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
self._assert_same_request(request, request2)
@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase):
self.assertEqual(r1.cookies, r2.cookies)
self.assertEqual(r1.meta, r2.meta)
self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
self.assertEqual(r1.encoding, r2.encoding)
self.assertEqual(r1._encoding, r2._encoding)
self.assertEqual(r1.priority, r2.priority)
self.assertEqual(r1.dont_filter, r2.dont_filter)
self.assertEqual(r1.flags, r2.flags)
if isinstance(r1, JsonRequest):
self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs)
def test_request_class(self):
r = FormRequest("http://www.example.com")
self._assert_serializes_ok(r, spider=self.spider)
r = CustomRequest("http://www.example.com")
self._assert_serializes_ok(r, spider=self.spider)
r1 = FormRequest("http://www.example.com")
self._assert_serializes_ok(r1, spider=self.spider)
r2 = CustomRequest("http://www.example.com")
self._assert_serializes_ok(r2, spider=self.spider)
r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
self._assert_serializes_ok(r3, spider=self.spider)
def test_callback_serialization(self):
r = Request("http://www.example.com", callback=self.spider.parse_item,
@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'], 'parse_item_reference')
self.assertEqual(request_dict['errback'], 'handle_error_reference')
@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase):
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
request_dict = r.to_dict(spider=self.spider)
self.assertEqual(request_dict['callback'],
'_TestSpider__parse_item_reference')
self.assertEqual(request_dict['errback'],
@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase):
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, request_to_dict, r)
self.assertRaises(ValueError, request_to_dict, r, spider=self.spider)
self.assertRaises(ValueError, r.to_dict, spider=self.spider)
def test_unserializable_callback2(self):
r = Request("http://www.example.com", callback=self.spider.parse_item)
self.assertRaises(ValueError, request_to_dict, r)
self.assertRaises(ValueError, r.to_dict, spider=None)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
name = 'my_spider'
def parse(self, response):
@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase):
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
setattr(spider, 'parse', None)
self.assertRaises(ValueError, request_to_dict, r, spider=spider)
self.assertRaises(ValueError, r.to_dict, spider=spider)
def test_callback_not_available(self):
"""Callback method is not available in the spider passed to from_dict"""
spider = TestSpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
def _assert_serializes_ok(self, request, spider=None):
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
with suppress(KeyError):
del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning
from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
request_copy = _from_dict(_to_dict(request, spider), spider)
self._assert_same_request(request, request_copy)
self.assertEqual(len(caught), 1)
self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
self.assertEqual(
"Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
" and/or scrapy.utils.request.request_from_dict instead",
str(caught[0].message),
)
class TestSpiderMixin:
@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin):
def __parse_item_private(self, response):
pass
class CustomRequest(Request):
pass

2
tests/test_utils_deprecate.py
View File

@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
warnings.simplefilter('ignore', MyWarning)
class UserClass(Deprecated):
pass

9
tests/test_utils_python.py
View File

@ -5,8 +5,9 @@ import platform
import unittest
from datetime import datetime
from itertools import count
from warnings import catch_warnings
from warnings import catch_warnings, filterwarnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
WeakKeyCache, get_func_args, to_bytes, to_unicode,
@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase):
pass
_values = count()
wk = WeakKeyCache(lambda k: next(_values))
with catch_warnings():
filterwarnings("ignore", category=ScrapyDeprecationWarning)
wk = WeakKeyCache(lambda k: next(_values))
k = _Weakme()
v = wk[k]
self.assertEqual(v, wk[k])

11
tox.ini
View File

@ -50,6 +50,8 @@ commands =
basepython = python3
deps =
{[testenv]deps}
# Twisted[http2] is required to import some files
Twisted[http2]>=17.9.0
pytest-flake8
commands =
py.test --flake8 {posargs:docs scrapy tests}
@ -57,12 +59,7 @@ commands =
[testenv:pylint]
basepython = python3
deps =
{[testenv]deps}
# Optional dependencies
boto
reppy
robotexclusionrulesparser
# Test dependencies
{[testenv:extra-deps]deps}
pylint
commands =
pylint conftest.py docs extras scrapy setup.py tests
@ -119,9 +116,11 @@ setenv =
[testenv:extra-deps]
deps =
{[testenv]deps}
boto
reppy
robotexclusionrulesparser
Pillow>=4.0.0
Twisted[http2]>=17.9.0
[testenv:asyncio]
commands =